FEATURE: Add support for Unicode usernames and group names

Co-authored-by: Joffrey JAFFEUX <j.jaffeux@gmail.com>

spec/components/pretty_text_spec.rb

@@ -336,6 +336,21 @@ describe PrettyText do
       expect(PrettyText.cook(". http://test/@sam")).not_to include('mention')
     end
 
+    context "with Unicode usernames disabled" do
+      before { SiteSetting.unicode_usernames = false }
+
+      it 'does not detect mention' do
+        expect(PrettyText.cook("Hello @狮子")).to_not include("mention")
+      end
+    end
+
+    context "with Unicode usernames enabled" do
+      before { SiteSetting.unicode_usernames = true }
+
+      it 'does detect mention' do
+        expect(PrettyText.cook("Hello @狮子")).to match_html '<p>Hello <span class="mention">@狮子</span></p>'
+      end
+    end
   end
 
   describe "code fences" do

spec/components/user_name_suggester_spec.rb

@@ -2,16 +2,10 @@ require 'rails_helper'
 require 'user_name_suggester'
 
 describe UserNameSuggester do
-
-  describe 'name heuristics' do
-    it 'is able to guess a decent username from an email' do
-      expect(UserNameSuggester.suggest('bob@bob.com')).to eq('bob')
-    end
-  end
-
   describe '.suggest' do
     before do
-      User.stubs(:username_length).returns(3..15)
+      SiteSetting.min_username_length = 3
+      SiteSetting.max_username_length = 15
     end
 
     it "doesn't raise an error on nil username" do
@@ -26,10 +20,6 @@ describe UserNameSuggester do
       expect(UserNameSuggester.suggest("Darth%^Vader")).to eq('Darth_Vader')
     end
 
-    it "transliterates some characters" do
-      expect(UserNameSuggester.suggest("Jørn")).to eq('Jorn')
-    end
-
     it 'adds 1 to an existing username' do
       user = Fabricate(:user)
       expect(UserNameSuggester.suggest(user.username)).to eq("#{user.username}1")
@@ -39,6 +29,10 @@ describe UserNameSuggester do
       expect(UserNameSuggester.suggest('a')).to eq('a11')
     end
 
+    it 'is able to guess a decent username from an email' do
+      expect(UserNameSuggester.suggest('bob@example.com')).to eq('bob')
+    end
+
     it "has a special case for me and i emails" do
       expect(UserNameSuggester.suggest('me@eviltrout.com')).to eq('eviltrout')
       expect(UserNameSuggester.suggest('i@eviltrout.com')).to eq('eviltrout')
@@ -106,6 +100,57 @@ describe UserNameSuggester do
       User.stubs(:username_length).returns(8..8)
       expect(UserNameSuggester.suggest('uuuuuuu_u')).to eq('uuuuuuu1')
     end
-  end
 
+    context "with Unicode usernames disabled" do
+      before { SiteSetting.unicode_usernames = false }
+
+      it "transliterates some characters" do
+        expect(UserNameSuggester.suggest('Jørn')).to eq('Jorn')
+      end
+
+      it "replaces Unicode characters" do
+        expect(UserNameSuggester.suggest('طائر')).to eq('111')
+        expect(UserNameSuggester.suggest('πουλί')).to eq('111')
+      end
+    end
+
+    context "with Unicode usernames enabled" do
+      before { SiteSetting.unicode_usernames = true }
+
+      it "does not transliterate" do
+        expect(UserNameSuggester.suggest("Jørn")).to eq('Jørn')
+      end
+
+      it "does not replace Unicode characters" do
+        expect(UserNameSuggester.suggest('طائر')).to eq('طائر')
+        expect(UserNameSuggester.suggest('πουλί')).to eq('πουλί')
+      end
+
+      it "shortens usernames by counting grapheme clusters" do
+        SiteSetting.max_username_length = 10
+        expect(UserNameSuggester.suggest('बहुत-लंबा-उपयोगकर्ता-नाम')).to eq('बहुत-लंबा-उपयो')
+      end
+
+      it "adds numbers if it's too short" do
+        expect(UserNameSuggester.suggest('鳥')).to eq('鳥11')
+
+        # grapheme cluster consists of 3 code points
+        expect(UserNameSuggester.suggest('য়া')).to eq('য়া11')
+      end
+
+      it "normalizes usernames" do
+        actual = 'Löwe'    # NFD, "Lo\u0308we"
+        expected = 'Löwe'  # NFC, "L\u00F6we"
+
+        expect(UserNameSuggester.suggest(actual)).to eq(expected)
+      end
+
+      it "does not suggest a username longer than max column size" do
+        SiteSetting.max_username_length = 40
+
+        expect(UserNameSuggester.suggest('য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া'))
+          .to eq('য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া-য়া')
+      end
+    end
+  end
 end

spec/components/validators/external_system_avatars_validator_spec.rb

@@ -0,0 +1,25 @@
+require 'rails_helper'
+
+describe ExternalSystemAvatarsValidator do
+  subject { described_class.new }
+
+  it "disallows disabling external system avatars when Unicode usernames are enabled" do
+    SiteSetting.unicode_usernames = true
+
+    expect(subject.valid_value?("f")).to eq(false)
+    expect(subject.error_message).to eq(I18n.t("site_settings.errors.unicode_usernames_avatars"))
+
+    expect(subject.valid_value?("t")).to eq(true)
+    expect(subject.error_message).to be_blank
+  end
+
+  it "allows disabling external system avatars when Unicode usernames are disabled" do
+    SiteSetting.unicode_usernames = false
+
+    expect(subject.valid_value?("t")).to eq(true)
+    expect(subject.error_message).to be_blank
+
+    expect(subject.valid_value?("f")).to eq(true)
+    expect(subject.error_message).to be_blank
+  end
+end

spec/components/validators/unicode_username_validator_spec.rb

@@ -0,0 +1,25 @@
+require 'rails_helper'
+
+describe UnicodeUsernameValidator do
+  subject { described_class.new }
+
+  it "disallows Unicode usernames when external system avatars are disabled" do
+    SiteSetting.external_system_avatars_enabled = false
+
+    expect(subject.valid_value?("t")).to eq(false)
+    expect(subject.error_message).to eq(I18n.t("site_settings.errors.unicode_usernames_avatars"))
+
+    expect(subject.valid_value?("f")).to eq(true)
+    expect(subject.error_message).to be_blank
+  end
+
+  it "allows Unicode usernames when external system avatars are enabled" do
+    SiteSetting.external_system_avatars_enabled = true
+
+    expect(subject.valid_value?("t")).to eq(true)
+    expect(subject.error_message).to be_blank
+
+    expect(subject.valid_value?("f")).to eq(true)
+    expect(subject.error_message).to be_blank
+  end
+end

spec/components/validators/unicode_username_whitelist_validator_spec.rb

@@ -0,0 +1,36 @@
+require 'rails_helper'
+
+describe UnicodeUsernameWhitelistValidator do
+  subject { described_class.new }
+
+  it "allows an empty whitelist" do
+    expect(subject.valid_value?("")).to eq(true)
+    expect(subject.error_message).to be_blank
+  end
+
+  it "disallows leading and trailing slashes" do
+    expected_error = I18n.t("site_settings.errors.unicode_username_whitelist.leading_trailing_slash")
+
+    expect(subject.valid_value?("/foo/")).to eq(false)
+    expect(subject.error_message).to eq(expected_error)
+
+    expect(subject.valid_value?("foo/")).to eq(true)
+    expect(subject.error_message).to be_blank
+
+    expect(subject.valid_value?("/foo")).to eq(true)
+    expect(subject.error_message).to be_blank
+
+    expect(subject.valid_value?("f/o/o")).to eq(true)
+    expect(subject.error_message).to be_blank
+
+    expect(subject.valid_value?("/foo/i")).to eq(false)
+    expect(subject.error_message).to eq(expected_error)
+  end
+
+  it "detects invalid regular expressions" do
+    expected_error = I18n.t("site_settings.errors.unicode_username_whitelist.regex_invalid", error: "")
+
+    expect(subject.valid_value?("\\p{Foo}")).to eq(false)
+    expect(subject.error_message).to start_with(expected_error)
+  end
+end