FIX: Use unaccent() for category name/slug search (#37622)

Categories with accented names (e.g. "Éditions") were not appearing in
search results, category chooser dropdowns, or hashtag autocomplete when
searching with unaccented terms (e.g. "editions").

PostgreSQL's `ILIKE` and `LOWER()` are case-insensitive but not
accent-insensitive, so queries like `name ILIKE '%editions%'` would not
match "Éditions".

This wraps all category name/slug comparisons with PostgreSQL's
`unaccent()` function across:
- `/categories/search` endpoint
- `/categories/hierarchical_search` endpoint
- Hashtag (#) autocomplete
- `category:` and `#slug` search filters

Introduces `Category.normalize_sql(expr)` helper that wraps SQL
expressions with `lower(unaccent(...))` to centralize the normalization
logic and make it easier to extend in the future.

On the frontend, adds a shared `removeAccents()` utility using NFD
normalization for client-side `Category.search()` and search filter
suggestions.

Ref - https://meta.discourse.org/t/395355

app/controllers/categories_controller.rb

@@ -403,7 +403,13 @@ class CategoriesController < ApplicationController
     categories = Category.secured(guardian)
 
     if term.present? && words = term.split
-      words.each { |word| categories = categories.where("name ILIKE ?", "%#{word}%") }
+      words.each do |word|
+        categories =
+          categories.where(
+            "#{Category.normalize_sql("name")} ILIKE #{Category.normalize_sql("?")}",
+            "%#{word}%",
+          )
+      end
     end
 
     categories =
@@ -441,7 +447,7 @@ class CategoriesController < ApplicationController
         .joins("LEFT JOIN topics t on t.id = categories.topic_id")
         .select("categories.*, t.slug topic_slug")
         .order(
-          "starts_with(lower(categories.name), #{ActiveRecord::Base.connection.quote(term)}) DESC",
+          "starts_with(#{Category.normalize_sql("categories.name")}, #{Category.normalize_sql(ActiveRecord::Base.connection.quote(term))}) DESC",
           "categories.parent_category_id IS NULL DESC",
           "categories.id IS NOT DISTINCT FROM #{ActiveRecord::Base.connection.quote(prioritized_category_id)} DESC",
           "categories.parent_category_id IS NOT DISTINCT FROM #{ActiveRecord::Base.connection.quote(prioritized_category_id)} DESC",

app/models/category.rb

@@ -232,6 +232,10 @@ class Category < ActiveRecord::Base
 
   enum :style_type, { square: 0, icon: 1, emoji: 2 }
 
+  def self.normalize_sql(expr)
+    "lower(unaccent(#{expr}))"
+  end
+
   def self.preload_user_fields!(guardian, categories)
     category_ids = categories.map(&:id)
 

app/services/category_hashtag_data_source.rb

@@ -79,12 +79,16 @@ class CategoryHashtagDataSource
         .includes(:parent_category)
 
     if condition == HashtagAutocompleteService.search_conditions[:starts_with]
-      base_search = base_search.where("starts_with(LOWER(slug), LOWER(:term))", term: term)
+      base_search =
+        base_search.where(
+          "starts_with(#{Category.normalize_sql("slug")}, #{Category.normalize_sql(":term")})",
+          term:,
+        )
     elsif condition == HashtagAutocompleteService.search_conditions[:contains]
       base_search =
         base_search.where(
-          "position(LOWER(:term) IN LOWER(name)) <> 0 OR position(LOWER(:term) IN LOWER(slug)) <> 0",
-          term: term,
+          "position(#{Category.normalize_sql(":term")} IN #{Category.normalize_sql("name")}) <> 0 OR position(#{Category.normalize_sql(":term")} IN #{Category.normalize_sql("slug")}) <> 0",
+          term:,
         )
     else
       raise Discourse::InvalidParameters.new("Unknown search condition: #{condition}")

app/services/category_hierarchical_search.rb

@@ -34,11 +34,11 @@ class CategoryHierarchicalSearch
       if query_params[:term].present?
         <<~SQL
           (
-            starts_with(LOWER(name), LOWER(:term))
+            starts_with(#{Category.normalize_sql("name")}, #{Category.normalize_sql(":term")})
             OR COALESCE(
               (
-                SELECT BOOL_AND(position(pattern IN LOWER(allowed_categories.name)) <> 0)
-                FROM unnest(regexp_split_to_array(LOWER(:term), '\\s+')) AS pattern
+                SELECT BOOL_AND(position(pattern IN #{Category.normalize_sql("allowed_categories.name")}) <> 0)
+                FROM unnest(regexp_split_to_array(#{Category.normalize_sql(":term")}, '\\s+')) AS pattern
               ),
               true
             )

frontend/discourse/app/lib/filter-suggestions.js

@@ -1,4 +1,5 @@
 import { ajax } from "discourse/lib/ajax";
+import { removeAccents } from "discourse/lib/utilities";
 import { i18n } from "discourse-i18n";
 
 const MAX_RESULTS = 20;
@@ -306,16 +307,17 @@ class FilterTypeValueSuggester {
 
   async getCategorySuggestions() {
     const categories = this.context.site?.categories || [];
-    const searchLower = this.searchTerm.toLowerCase();
+    const normalize = (str) => removeAccents(str.toLowerCase());
+    const searchNormalized = normalize(this.searchTerm);
 
     return categories
       .filter((cat) => {
-        const name = cat.name.toLowerCase();
-        const slug = cat.slug.toLowerCase();
+        const name = normalize(cat.name);
+        const slug = normalize(cat.slug);
         return (
-          !searchLower ||
-          name.includes(searchLower) ||
-          slug.includes(searchLower)
+          !searchNormalized ||
+          name.includes(searchNormalized) ||
+          slug.includes(searchNormalized)
         );
       })
       .slice(0, 10)

frontend/discourse/app/lib/utilities.js

@@ -338,12 +338,16 @@ export function clipboardHelpers(e, opts) {
   return { clipboard, types, canUpload, canPasteHtml };
 }
 
+export function removeAccents(string) {
+  return string.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+}
+
 // Replace any accented characters with their ASCII equivalent
 // Return the string if it only contains ASCII printable characters,
 // otherwise use the fallback
 export function toAsciiPrintable(string, fallback) {
   if (typeof string.normalize === "function") {
-    string = string.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+    string = removeAccents(string);
   }
 
   return /^[\040-\176]*$/.test(string) ? string : fallback;

frontend/discourse/app/models/category.js

@@ -18,6 +18,7 @@ import { MultiCache } from "discourse/lib/multi-cache";
 import { NotificationLevels } from "discourse/lib/notification-levels";
 import { trackedArray } from "discourse/lib/tracked-tools";
 import { applyValueTransformer } from "discourse/lib/transformer";
+import { removeAccents } from "discourse/lib/utilities";
 import PermissionType from "discourse/models/permission-type";
 import RestModel from "discourse/models/rest";
 import Site from "discourse/models/site";
@@ -353,8 +354,10 @@ export default class Category extends RestModel {
     const emptyTerm = term === "";
     let slugTerm = term;
 
+    const normalize = (str) => removeAccents(str.toLowerCase());
+
     if (!emptyTerm) {
-      term = term.toLowerCase();
+      term = normalize(term);
       slugTerm = term;
       term = term.replace(/-/g, " ");
     }
@@ -380,8 +383,8 @@ export default class Category extends RestModel {
       if (
         ((emptyTerm && !category.get("parent_category_id")) ||
           (!emptyTerm &&
-            (category.get("name").toLowerCase().startsWith(term) ||
-              category.get("slug").toLowerCase().startsWith(slugTerm)))) &&
+            (normalize(category.get("name")).startsWith(term) ||
+              normalize(category.get("slug")).startsWith(slugTerm)))) &&
         validCategoryParent(category)
       ) {
         data.push(category);
@@ -393,9 +396,8 @@ export default class Category extends RestModel {
         const category = categories[i];
 
         if (
-          ((!emptyTerm &&
-            category.get("name").toLowerCase().indexOf(term) > 0) ||
-            category.get("slug").toLowerCase().indexOf(slugTerm) > 0) &&
+          ((!emptyTerm && normalize(category.get("name")).indexOf(term) > 0) ||
+            normalize(category.get("slug")).indexOf(slugTerm) > 0) &&
           validCategoryParent(category)
         ) {
           if (!data.includes(category)) {

frontend/discourse/tests/unit/models/category-test.js

@@ -414,6 +414,38 @@ module("Unit | Model | category", function (hooks) {
     );
   });
 
+  test("search with accented category name", function (assert) {
+    const store = getOwner(this).lookup("service:store");
+    const category1 = store.createRecord("category", {
+      id: 1,
+      name: "Éditions",
+      slug: "publications",
+    });
+    const category2 = store.createRecord("category", {
+      id: 2,
+      name: "Cinéma",
+      slug: "films",
+    });
+
+    sinon.stub(Category, "listByActivity").returns([category1, category2]);
+
+    assert.deepEqual(
+      Category.search("editions"),
+      [category1],
+      "finds accented category with unaccented search term"
+    );
+    assert.deepEqual(
+      Category.search("cinema"),
+      [category2],
+      "finds accented category by unaccented name"
+    );
+    assert.deepEqual(
+      Category.search("Édit"),
+      [category1],
+      "finds accented category with accented search term"
+    );
+  });
+
   test("search with category slug", function (assert) {
     const store = getOwner(this).lookup("service:store");
     const category1 = store.createRecord("category", {

lib/search.rb

@@ -608,8 +608,8 @@ class Search
           categories c
       JOIN
           unnest(ARRAY[:matches]) AS term ON
-          c.slug ILIKE term OR
-          c.name ILIKE term OR
+          #{Category.normalize_sql("c.slug")} ILIKE #{Category.normalize_sql("term")} OR
+          #{Category.normalize_sql("c.name")} ILIKE #{Category.normalize_sql("term")} OR
           (term ~ '^[0-9]{1,10}$' AND c.id = term::int)
       SQL
 
@@ -645,15 +645,24 @@ class Search
     category_id =
       if subcategory_slug
         Category
-          .where("lower(slug) = ?", subcategory_slug.downcase)
+          .where(
+            "#{Category.normalize_sql("slug")} = #{Category.normalize_sql("?")}",
+            subcategory_slug,
+          )
           .where(
             parent_category_id:
-              Category.where("lower(slug) = ?", category_slug.downcase).select(:id),
+              Category.where(
+                "#{Category.normalize_sql("slug")} = #{Category.normalize_sql("?")}",
+                category_slug,
+              ).select(:id),
           )
           .pick(:id)
       else
         Category
-          .where("lower(slug) = ?", category_slug.downcase)
+          .where(
+            "#{Category.normalize_sql("slug")} = #{Category.normalize_sql("?")}",
+            category_slug,
+          )
           .order("case when parent_category_id is null then 0 else 1 end")
           .pick(:id)
       end

spec/lib/search_spec.rb

@@ -1655,6 +1655,16 @@ RSpec.describe Search do
       expect(search.posts).to eq([ignored_category.topic.first_post])
     end
 
+    it "matches categories with accented names using category: filter" do
+      accented_category =
+        Fabricate(:category_with_definition, name: "Éditions", slug: "publications")
+      topic_in_accented = Fabricate(:topic, category: accented_category)
+      post_in_accented = Fabricate(:post, topic: topic_in_accented, raw: "snow monkey")
+
+      search = Search.execute("monkey category:Editions")
+      expect(search.posts.map(&:id)).to include(post_in_accented.id)
+    end
+
     describe "with child categories" do
       let!(:child_of_ignored_category) do
         Fabricate(

spec/requests/categories_controller_spec.rb

@@ -1593,6 +1593,22 @@ RSpec.describe CategoriesController do
           "Notfoo",
         )
       end
+
+      it "matches categories with accented names using unaccented search term" do
+        accented_category = Fabricate(:category, name: "Éditions")
+
+        post "/categories/search.json", params: { term: "editions" }
+
+        expect(response.parsed_body["categories"].map { |c| c["name"] }).to include("Éditions")
+      end
+
+      it "matches categories with unaccented names using accented search term" do
+        Fabricate(:category, name: "Editions")
+
+        post "/categories/search.json", params: { term: "Éditions" }
+
+        expect(response.parsed_body["categories"].map { |c| c["name"] }).to include("Editions")
+      end
     end
 
     context "with parent_category_id" do

spec/services/category_hashtag_data_source_spec.rb

@@ -138,6 +138,21 @@ RSpec.describe CategoryHashtagDataSource do
       expect(result.slug).to eq("books")
     end
 
+    it "finds categories with accented names using unaccented search term" do
+      Fabricate(:category, name: "Cinéma", slug: "films", topic_count: 10)
+
+      result = described_class.search(guardian, "cinema", 5)
+      expect(result.map(&:text)).to include("Cinéma")
+    end
+
+    it "finds categories with accented slugs using unaccented search term" do
+      SiteSetting.slug_generation_method = "none"
+      Fabricate(:category, name: "Cinéma", slug: "cinéma", topic_count: 10)
+
+      result = described_class.search(guardian, "cinema", 5)
+      expect(result.map(&:text)).to include("Cinéma")
+    end
+
     it "does not find categories the user cannot access" do
       expect(described_class.search(guardian, "secret", 5).first).to eq(nil)
       group.add(user)

spec/services/category_hierarchical_search_spec.rb

@@ -87,6 +87,15 @@ RSpec.describe CategoryHierarchicalSearch do
     )
   end
 
+  it "matches categories with accented names using unaccented search term" do
+    Fabricate(:category, name: "Éditions spéciales")
+
+    context = described_class.call(guardian: Guardian.new, params: { term: "editions speciales" })
+
+    expect(context).to be_success
+    expect(context.categories.map(&:name)).to include("Éditions spéciales")
+  end
+
   it "returns categories with their ancestors that match the terms param in a hierarchical order" do
     context = described_class.call(guardian: Guardian.new, params: { term: "match" })