From 1a2f9f311e487c87ad57f0720ee3f42ec0760911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Thu, 12 Feb 2026 08:43:23 +0100 Subject: [PATCH] FIX: Use unaccent() for category name/slug search (#37622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Categories with accented names (e.g. "Éditions") were not appearing in search results, category chooser dropdowns, or hashtag autocomplete when searching with unaccented terms (e.g. "editions"). PostgreSQL's `ILIKE` and `LOWER()` are case-insensitive but not accent-insensitive, so queries like `name ILIKE '%editions%'` would not match "Éditions". This wraps all category name/slug comparisons with PostgreSQL's `unaccent()` function across: - `/categories/search` endpoint - `/categories/hierarchical_search` endpoint - Hashtag (#) autocomplete - `category:` and `#slug` search filters Introduces `Category.normalize_sql(expr)` helper that wraps SQL expressions with `lower(unaccent(...))` to centralize the normalization logic and make it easier to extend in the future. On the frontend, adds a shared `removeAccents()` utility using NFD normalization for client-side `Category.search()` and search filter suggestions. Ref - https://meta.discourse.org/t/395355 --- app/controllers/categories_controller.rb | 10 ++++-- app/models/category.rb | 4 +++ app/services/category_hashtag_data_source.rb | 10 ++++-- app/services/category_hierarchical_search.rb | 6 ++-- .../discourse/app/lib/filter-suggestions.js | 14 ++++---- frontend/discourse/app/lib/utilities.js | 6 +++- frontend/discourse/app/models/category.js | 14 ++++---- .../tests/unit/models/category-test.js | 32 +++++++++++++++++++ lib/search.rb | 19 ++++++++--- spec/lib/search_spec.rb | 10 ++++++ spec/requests/categories_controller_spec.rb | 16 ++++++++++ .../category_hashtag_data_source_spec.rb | 15 +++++++++ .../category_hierarchical_search_spec.rb | 9 ++++++ 13 files changed, 139 insertions(+), 26 deletions(-) diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index 624fffdbf07..055ae9d3e7c 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -403,7 +403,13 @@ class CategoriesController < ApplicationController categories = Category.secured(guardian) if term.present? && words = term.split - words.each { |word| categories = categories.where("name ILIKE ?", "%#{word}%") } + words.each do |word| + categories = + categories.where( + "#{Category.normalize_sql("name")} ILIKE #{Category.normalize_sql("?")}", + "%#{word}%", + ) + end end categories = @@ -441,7 +447,7 @@ class CategoriesController < ApplicationController .joins("LEFT JOIN topics t on t.id = categories.topic_id") .select("categories.*, t.slug topic_slug") .order( - "starts_with(lower(categories.name), #{ActiveRecord::Base.connection.quote(term)}) DESC", + "starts_with(#{Category.normalize_sql("categories.name")}, #{Category.normalize_sql(ActiveRecord::Base.connection.quote(term))}) DESC", "categories.parent_category_id IS NULL DESC", "categories.id IS NOT DISTINCT FROM #{ActiveRecord::Base.connection.quote(prioritized_category_id)} DESC", "categories.parent_category_id IS NOT DISTINCT FROM #{ActiveRecord::Base.connection.quote(prioritized_category_id)} DESC", diff --git a/app/models/category.rb b/app/models/category.rb index a6ae9102031..f976e67cb08 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -232,6 +232,10 @@ class Category < ActiveRecord::Base enum :style_type, { square: 0, icon: 1, emoji: 2 } + def self.normalize_sql(expr) + "lower(unaccent(#{expr}))" + end + def self.preload_user_fields!(guardian, categories) category_ids = categories.map(&:id) diff --git a/app/services/category_hashtag_data_source.rb b/app/services/category_hashtag_data_source.rb index fd0be8541fe..cd5a39de5f1 100644 --- a/app/services/category_hashtag_data_source.rb +++ b/app/services/category_hashtag_data_source.rb @@ -79,12 +79,16 @@ class CategoryHashtagDataSource .includes(:parent_category) if condition == HashtagAutocompleteService.search_conditions[:starts_with] - base_search = base_search.where("starts_with(LOWER(slug), LOWER(:term))", term: term) + base_search = + base_search.where( + "starts_with(#{Category.normalize_sql("slug")}, #{Category.normalize_sql(":term")})", + term:, + ) elsif condition == HashtagAutocompleteService.search_conditions[:contains] base_search = base_search.where( - "position(LOWER(:term) IN LOWER(name)) <> 0 OR position(LOWER(:term) IN LOWER(slug)) <> 0", - term: term, + "position(#{Category.normalize_sql(":term")} IN #{Category.normalize_sql("name")}) <> 0 OR position(#{Category.normalize_sql(":term")} IN #{Category.normalize_sql("slug")}) <> 0", + term:, ) else raise Discourse::InvalidParameters.new("Unknown search condition: #{condition}") diff --git a/app/services/category_hierarchical_search.rb b/app/services/category_hierarchical_search.rb index e2c0c4ead32..412eac1f965 100644 --- a/app/services/category_hierarchical_search.rb +++ b/app/services/category_hierarchical_search.rb @@ -34,11 +34,11 @@ class CategoryHierarchicalSearch if query_params[:term].present? <<~SQL ( - starts_with(LOWER(name), LOWER(:term)) + starts_with(#{Category.normalize_sql("name")}, #{Category.normalize_sql(":term")}) OR COALESCE( ( - SELECT BOOL_AND(position(pattern IN LOWER(allowed_categories.name)) <> 0) - FROM unnest(regexp_split_to_array(LOWER(:term), '\\s+')) AS pattern + SELECT BOOL_AND(position(pattern IN #{Category.normalize_sql("allowed_categories.name")}) <> 0) + FROM unnest(regexp_split_to_array(#{Category.normalize_sql(":term")}, '\\s+')) AS pattern ), true ) diff --git a/frontend/discourse/app/lib/filter-suggestions.js b/frontend/discourse/app/lib/filter-suggestions.js index c471eb85015..082345494eb 100644 --- a/frontend/discourse/app/lib/filter-suggestions.js +++ b/frontend/discourse/app/lib/filter-suggestions.js @@ -1,4 +1,5 @@ import { ajax } from "discourse/lib/ajax"; +import { removeAccents } from "discourse/lib/utilities"; import { i18n } from "discourse-i18n"; const MAX_RESULTS = 20; @@ -306,16 +307,17 @@ class FilterTypeValueSuggester { async getCategorySuggestions() { const categories = this.context.site?.categories || []; - const searchLower = this.searchTerm.toLowerCase(); + const normalize = (str) => removeAccents(str.toLowerCase()); + const searchNormalized = normalize(this.searchTerm); return categories .filter((cat) => { - const name = cat.name.toLowerCase(); - const slug = cat.slug.toLowerCase(); + const name = normalize(cat.name); + const slug = normalize(cat.slug); return ( - !searchLower || - name.includes(searchLower) || - slug.includes(searchLower) + !searchNormalized || + name.includes(searchNormalized) || + slug.includes(searchNormalized) ); }) .slice(0, 10) diff --git a/frontend/discourse/app/lib/utilities.js b/frontend/discourse/app/lib/utilities.js index d68ef5438b6..793c42f4966 100644 --- a/frontend/discourse/app/lib/utilities.js +++ b/frontend/discourse/app/lib/utilities.js @@ -338,12 +338,16 @@ export function clipboardHelpers(e, opts) { return { clipboard, types, canUpload, canPasteHtml }; } +export function removeAccents(string) { + return string.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); +} + // Replace any accented characters with their ASCII equivalent // Return the string if it only contains ASCII printable characters, // otherwise use the fallback export function toAsciiPrintable(string, fallback) { if (typeof string.normalize === "function") { - string = string.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); + string = removeAccents(string); } return /^[\040-\176]*$/.test(string) ? string : fallback; diff --git a/frontend/discourse/app/models/category.js b/frontend/discourse/app/models/category.js index 8e3bb9c6a88..fb43284031c 100644 --- a/frontend/discourse/app/models/category.js +++ b/frontend/discourse/app/models/category.js @@ -18,6 +18,7 @@ import { MultiCache } from "discourse/lib/multi-cache"; import { NotificationLevels } from "discourse/lib/notification-levels"; import { trackedArray } from "discourse/lib/tracked-tools"; import { applyValueTransformer } from "discourse/lib/transformer"; +import { removeAccents } from "discourse/lib/utilities"; import PermissionType from "discourse/models/permission-type"; import RestModel from "discourse/models/rest"; import Site from "discourse/models/site"; @@ -353,8 +354,10 @@ export default class Category extends RestModel { const emptyTerm = term === ""; let slugTerm = term; + const normalize = (str) => removeAccents(str.toLowerCase()); + if (!emptyTerm) { - term = term.toLowerCase(); + term = normalize(term); slugTerm = term; term = term.replace(/-/g, " "); } @@ -380,8 +383,8 @@ export default class Category extends RestModel { if ( ((emptyTerm && !category.get("parent_category_id")) || (!emptyTerm && - (category.get("name").toLowerCase().startsWith(term) || - category.get("slug").toLowerCase().startsWith(slugTerm)))) && + (normalize(category.get("name")).startsWith(term) || + normalize(category.get("slug")).startsWith(slugTerm)))) && validCategoryParent(category) ) { data.push(category); @@ -393,9 +396,8 @@ export default class Category extends RestModel { const category = categories[i]; if ( - ((!emptyTerm && - category.get("name").toLowerCase().indexOf(term) > 0) || - category.get("slug").toLowerCase().indexOf(slugTerm) > 0) && + ((!emptyTerm && normalize(category.get("name")).indexOf(term) > 0) || + normalize(category.get("slug")).indexOf(slugTerm) > 0) && validCategoryParent(category) ) { if (!data.includes(category)) { diff --git a/frontend/discourse/tests/unit/models/category-test.js b/frontend/discourse/tests/unit/models/category-test.js index 9244c8f6ab1..6b94839114b 100644 --- a/frontend/discourse/tests/unit/models/category-test.js +++ b/frontend/discourse/tests/unit/models/category-test.js @@ -414,6 +414,38 @@ module("Unit | Model | category", function (hooks) { ); }); + test("search with accented category name", function (assert) { + const store = getOwner(this).lookup("service:store"); + const category1 = store.createRecord("category", { + id: 1, + name: "Éditions", + slug: "publications", + }); + const category2 = store.createRecord("category", { + id: 2, + name: "Cinéma", + slug: "films", + }); + + sinon.stub(Category, "listByActivity").returns([category1, category2]); + + assert.deepEqual( + Category.search("editions"), + [category1], + "finds accented category with unaccented search term" + ); + assert.deepEqual( + Category.search("cinema"), + [category2], + "finds accented category by unaccented name" + ); + assert.deepEqual( + Category.search("Édit"), + [category1], + "finds accented category with accented search term" + ); + }); + test("search with category slug", function (assert) { const store = getOwner(this).lookup("service:store"); const category1 = store.createRecord("category", { diff --git a/lib/search.rb b/lib/search.rb index 175d3b78659..245b22fcbe0 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -608,8 +608,8 @@ class Search categories c JOIN unnest(ARRAY[:matches]) AS term ON - c.slug ILIKE term OR - c.name ILIKE term OR + #{Category.normalize_sql("c.slug")} ILIKE #{Category.normalize_sql("term")} OR + #{Category.normalize_sql("c.name")} ILIKE #{Category.normalize_sql("term")} OR (term ~ '^[0-9]{1,10}$' AND c.id = term::int) SQL @@ -645,15 +645,24 @@ class Search category_id = if subcategory_slug Category - .where("lower(slug) = ?", subcategory_slug.downcase) + .where( + "#{Category.normalize_sql("slug")} = #{Category.normalize_sql("?")}", + subcategory_slug, + ) .where( parent_category_id: - Category.where("lower(slug) = ?", category_slug.downcase).select(:id), + Category.where( + "#{Category.normalize_sql("slug")} = #{Category.normalize_sql("?")}", + category_slug, + ).select(:id), ) .pick(:id) else Category - .where("lower(slug) = ?", category_slug.downcase) + .where( + "#{Category.normalize_sql("slug")} = #{Category.normalize_sql("?")}", + category_slug, + ) .order("case when parent_category_id is null then 0 else 1 end") .pick(:id) end diff --git a/spec/lib/search_spec.rb b/spec/lib/search_spec.rb index 7757264c6c7..7f1efce2c08 100644 --- a/spec/lib/search_spec.rb +++ b/spec/lib/search_spec.rb @@ -1655,6 +1655,16 @@ RSpec.describe Search do expect(search.posts).to eq([ignored_category.topic.first_post]) end + it "matches categories with accented names using category: filter" do + accented_category = + Fabricate(:category_with_definition, name: "Éditions", slug: "publications") + topic_in_accented = Fabricate(:topic, category: accented_category) + post_in_accented = Fabricate(:post, topic: topic_in_accented, raw: "snow monkey") + + search = Search.execute("monkey category:Editions") + expect(search.posts.map(&:id)).to include(post_in_accented.id) + end + describe "with child categories" do let!(:child_of_ignored_category) do Fabricate( diff --git a/spec/requests/categories_controller_spec.rb b/spec/requests/categories_controller_spec.rb index 5aed4d2c93c..0e177cceb7a 100644 --- a/spec/requests/categories_controller_spec.rb +++ b/spec/requests/categories_controller_spec.rb @@ -1593,6 +1593,22 @@ RSpec.describe CategoriesController do "Notfoo", ) end + + it "matches categories with accented names using unaccented search term" do + accented_category = Fabricate(:category, name: "Éditions") + + post "/categories/search.json", params: { term: "editions" } + + expect(response.parsed_body["categories"].map { |c| c["name"] }).to include("Éditions") + end + + it "matches categories with unaccented names using accented search term" do + Fabricate(:category, name: "Editions") + + post "/categories/search.json", params: { term: "Éditions" } + + expect(response.parsed_body["categories"].map { |c| c["name"] }).to include("Editions") + end end context "with parent_category_id" do diff --git a/spec/services/category_hashtag_data_source_spec.rb b/spec/services/category_hashtag_data_source_spec.rb index 12fb7820d43..e105c08003b 100644 --- a/spec/services/category_hashtag_data_source_spec.rb +++ b/spec/services/category_hashtag_data_source_spec.rb @@ -138,6 +138,21 @@ RSpec.describe CategoryHashtagDataSource do expect(result.slug).to eq("books") end + it "finds categories with accented names using unaccented search term" do + Fabricate(:category, name: "Cinéma", slug: "films", topic_count: 10) + + result = described_class.search(guardian, "cinema", 5) + expect(result.map(&:text)).to include("Cinéma") + end + + it "finds categories with accented slugs using unaccented search term" do + SiteSetting.slug_generation_method = "none" + Fabricate(:category, name: "Cinéma", slug: "cinéma", topic_count: 10) + + result = described_class.search(guardian, "cinema", 5) + expect(result.map(&:text)).to include("Cinéma") + end + it "does not find categories the user cannot access" do expect(described_class.search(guardian, "secret", 5).first).to eq(nil) group.add(user) diff --git a/spec/services/category_hierarchical_search_spec.rb b/spec/services/category_hierarchical_search_spec.rb index ed40a48d19c..98aba3b28a4 100644 --- a/spec/services/category_hierarchical_search_spec.rb +++ b/spec/services/category_hierarchical_search_spec.rb @@ -87,6 +87,15 @@ RSpec.describe CategoryHierarchicalSearch do ) end + it "matches categories with accented names using unaccented search term" do + Fabricate(:category, name: "Éditions spéciales") + + context = described_class.call(guardian: Guardian.new, params: { term: "editions speciales" }) + + expect(context).to be_success + expect(context.categories.map(&:name)).to include("Éditions spéciales") + end + it "returns categories with their ancestors that match the terms param in a hierarchical order" do context = described_class.call(guardian: Guardian.new, params: { term: "match" })