PLT-2816 Fixed handling of Unicode 8 emojis (#2924)

* Updated twemoji to properly recognize Unicode 8.0 emojis

* Updated unicode emoji parser to only render emojis we support as images

* Corrected filename for South African flag emoji

* Added Mattermost emoticons!

* Added additional emoticons to test files

tests/test-emoticons1.md

@@ -1,12 +1,14 @@
 # Emoticon Testing  
 Verify that all emoticons render. This test should render in three separate messages since it's ~11000 characters.  
 
+:mm: :mattermost:
+
 ### Emoticon - Punctuation  
 
 :) :-) ;) ;-) :o :O :-o :-O :] :-] :d :-D x-d x-D :p :-P :@ :( :-(  :'( :/ :-/ :s :-s :| :-| :$ :-$ :-x <3 :+1: :-1:  
 
 ### Emoticons - People  
-:bowtie: :smile: :laughing: :blush: :smiley: :relaxed: :smirk: :heart_eyes: :kissing_heart: :kissing_closed_eyes: :flushed: :relieved: :satisfied: :grin: :wink: :stuck_out_tongue_winking_eye: :stuck_out_tongue_closed_eyes: :grinning: :kissing: :kissing_smiling_eyes: :stuck_out_tongue: :sleeping: :worried: :frowning: :anguished: :open_mouth: :grimacing: :confused: :hushed: :expressionless: :unamused: :sweat_smile: :sweat: :disappointed_relieved: :weary: :pensive: :disappointed: :confounded: :fearful: :cold_sweat: :persevere: :cry: :sob: :joy: :astonished: :scream: :neckbeard: :tired_face: :angry: :rage: :triumph: :sleepy: :yum: :mask: :sunglasses: :dizzy_face: :imp: :smiling_imp: :neutral_face: :no_mouth: :innocent: :alien: :yellow_heart: :blue_heart: :purple_heart: :heart: :green_heart: :broken_heart: :heartbeat: :heartpulse: :two_hearts: :revolving_hearts: :cupid: :sparkling_heart: :sparkles: :star: :star2: :dizzy: :boom: :collision: :anger: :exclamation: :question: :grey_exclamation: :grey_question: :zzz: :dash: :sweat_drops: :notes: :musical_note: :fire: :hankey: :poop: :shit: :+1: :thumbsup: :-1: :thumbsdown: :ok_hand: :punch: :facepunch: :fist: :v: :wave: :hand: :raised_hand: :open_hands: :point_up: :point_down: :point_left: :point_right: :raised_hands: :pray: :point_up_2: :clap: :muscle: :metal: :fu: :runner: :running: :couple: :family: :two_men_holding_hands: :two_women_holding_hands: :dancer: :dancers: :ok_woman: :no_good: :information_desk_person: :raising_hand: :bride_with_veil: :person_with_pouting_face: :person_frowning: :bow: :couplekiss: :couple_with_heart: :massage: :haircut: :nail_care: :boy: :girl: :woman: :man: :baby: :older_woman: :older_man: :person_with_blond_hair: :man_with_gua_pi_mao: :man_with_turban: :construction_worker: :cop: :angel: :princess: :smiley_cat: :smile_cat: :heart_eyes_cat: :kissing_cat: :smirk_cat: :scream_cat: :crying_cat_face: :joy_cat: :pouting_cat: :japanese_ogre: :japanese_goblin: :see_no_evil: :hear_no_evil: :speak_no_evil: :guardsman: :skull: :feet: :lips: :kiss: :droplet: :ear: :eyes: :nose: :tongue: :love_letter: :bust_in_silhouette: :busts_in_silhouette: :speech_balloon: :thought_balloon: :feelsgood: :finnadie: :goberserk: :godmode: :hurtrealbad: :rage1: :rage2: :rage3: :rage4: :suspect: :trollface:  
+:bowtie: :smile: :laughing: :blush: :smiley: :relaxed: :smirk: :heart_eyes: :kissing_heart: :kissing_closed_eyes: :flushed: :relieved: :satisfied: :grin: :wink: :stuck_out_tongue_winking_eye: :stuck_out_tongue_closed_eyes: :grinning: :kissing: :kissing_smiling_eyes: :stuck_out_tongue: :sleeping: :worried: :frowning: :anguished: :open_mouth: :grimacing: :confused: :hushed: :expressionless: :unamused: :sweat_smile: :sweat: :disappointed_relieved: :weary: :pensive: :disappointed: :confounded: :fearful: :cold_sweat: :persevere: :cry: :sob: :joy: :astonished: :scream: :neckbeard: :tired_face: :angry: :rage: :triumph: :sleepy: :yum: :mask: :sunglasses: :dizzy_face: :imp: :smiling_imp: :neutral_face: :no_mouth: :innocent: :alien: :yellow_heart: :blue_heart: :purple_heart: :heart: :green_heart: :broken_heart: :heartbeat: :heartpulse: :two_hearts: :revolving_hearts: :cupid: :sparkling_heart: :sparkles: :star: :star2: :dizzy: :boom: :collision: :anger: :exclamation: :question: :grey_exclamation: :grey_question: :zzz: :dash: :sweat_drops: :notes: :musical_note: :fire: :hankey: :poop: :shit: :+1: :thumbsup: :-1: :thumbsdown: :ok_hand: :punch: :facepunch: :fist: :v: :wave: :hand: :raised_hand: :open_hands: :point_up: :point_down: :point_left: :point_right: :raised_hands: :pray: :point_up_2: :clap: :muscle: :metal: :fu: :runner: :running: :couple: :family: :two_men_holding_hands: :two_women_holding_hands: :dancer: :dancers: :ok_woman: :no_good: :information_desk_person: :raising_hand: :bride_with_veil: :person_with_pouting_face: :person_frowning: :bow: :couplekiss: :couple_with_heart: :massage: :haircut: :nail_care: :boy: :girl: :woman: :man: :baby: :older_woman: :older_man: :person_with_blond_hair: :man_with_gua_pi_mao: :man_with_turban: :construction_worker: :cop: :angel: :princess: :smiley_cat: :smile_cat: :heart_eyes_cat: :kissing_cat: :smirk_cat: :scream_cat: :crying_cat_face: :joy_cat: :pouting_cat: :japanese_ogre: :japanese_goblin: :see_no_evil: :hear_no_evil: :speak_no_evil: :guardsman: :skull: :feet: :lips: :kiss: :droplet: :ear: :eyes: :nose: :tongue: :love_letter: :bust_in_silhouette: :busts_in_silhouette: :speech_balloon: :thought_balloon: :feelsgood: :finnadie: :goberserk: :godmode: :hurtrealbad: :rage1: :rage2: :rage3: :rage4: :suspect: :trollface: :slightly_smiling_face: :slightly_frowning_face: :upside_down_face:  
 
 ### Emoticons - Nature  
 :sunny: :umbrella: :cloud: :snowflake: :snowman: :zap: :cyclone: :foggy: :ocean: :cat: :dog: :mouse: :hamster: :rabbit: :wolf: :frog: :tiger: :koala: :bear: :pig: :pig_nose: :cow: :boar: :monkey_face: :monkey: :horse: :racehorse: :camel: :sheep: :elephant: :panda_face: :snake: :bird: :baby_chick: :hatched_chick: :hatching_chick: :chicken: :penguin: :turtle: :bug: :honeybee: :ant: :beetle: :snail: :octopus: :tropical_fish: :fish: :whale: :whale2: :dolphin: :cow2: :ram: :rat: :water_buffalo: :tiger2: :rabbit2: :dragon: :goat: :rooster: :dog2: :pig2: :mouse2: :ox: :dragon_face: :blowfish: :crocodile: :dromedary_camel: :leopard: :cat2: :poodle: :paw_prints: :bouquet: :cherry_blossom: :tulip: :four_leaf_clover: :rose: :sunflower: :hibiscus: :maple_leaf: :leaves: :fallen_leaf: :herb: :mushroom: :cactus: :palm_tree: :evergreen_tree: :deciduous_tree: :chestnut: :seedling: :blossom: :ear_of_rice: :shell: :globe_with_meridians: :sun_with_face: :full_moon_with_face: :new_moon_with_face: :new_moon: :waxing_crescent_moon: :first_quarter_moon: :waxing_gibbous_moon: :full_moon: :waning_gibbous_moon: :last_quarter_moon: :waning_crescent_moon: :last_quarter_moon_with_face: :first_quarter_moon_with_face: :crescent_moon: :earth_africa: :earth_americas: :earth_asia: :volcano: :milky_way: :partly_sunny: :octocat: :squirrel:  

tests/test-emoticons3.md

@@ -1,3 +1,3 @@
 ### Emoticons - Places     
 
-:house: :house_with_garden: :school: :office: :post_office: :hospital: :bank: :convenience_store: :love_hotel: :hotel: :wedding: :church: :department_store: :european_post_office: :city_sunrise: :city_sunset: :japanese_castle: :european_castle: :tent: :factory: :tokyo_tower: :japan: :mount_fuji: :sunrise_over_mountains: :sunrise: :stars: :statue_of_liberty: :bridge_at_night: :carousel_horse: :rainbow: :ferris_wheel: :fountain: :roller_coaster: :ship: :speedboat: :boat: :sailboat: :rowboat: :anchor: :rocket: :airplane: :helicopter: :steam_locomotive: :tram: :mountain_railway: :bike: :aerial_tramway: :suspension_railway: :mountain_cableway: :tractor: :blue_car: :oncoming_automobile: :car: :red_car: :taxi: :oncoming_taxi: :articulated_lorry: :bus: :oncoming_bus: :rotating_light: :police_car: :oncoming_police_car: :fire_engine: :ambulance: :minibus: :truck: :train: :station: :train2: :bullettrain_front: :bullettrain_side: :light_rail: :monorail: :railway_car: :trolleybus: :ticket: :fuelpump: :vertical_traffic_light: :traffic_light: :warning: :construction: :beginner: :atm: :slot_machine: :busstop: :barber: :hotsprings: :checkered_flag: :crossed_flags: :izakaya_lantern: :moyai: :circus_tent: :performing_arts: :round_pushpin: :triangular_flag_on_post: :jp: :kr: :cn: :us: :fr: :es: :it: :ru: :gb: :uk: :de:  
+:house: :house_with_garden: :school: :office: :post_office: :hospital: :bank: :convenience_store: :love_hotel: :hotel: :wedding: :church: :department_store: :european_post_office: :city_sunrise: :city_sunset: :japanese_castle: :european_castle: :tent: :factory: :tokyo_tower: :japan: :mount_fuji: :sunrise_over_mountains: :sunrise: :stars: :statue_of_liberty: :bridge_at_night: :carousel_horse: :rainbow: :ferris_wheel: :fountain: :roller_coaster: :ship: :speedboat: :boat: :sailboat: :rowboat: :anchor: :rocket: :airplane: :helicopter: :steam_locomotive: :tram: :mountain_railway: :bike: :aerial_tramway: :suspension_railway: :mountain_cableway: :tractor: :blue_car: :oncoming_automobile: :car: :red_car: :taxi: :oncoming_taxi: :articulated_lorry: :bus: :oncoming_bus: :rotating_light: :police_car: :oncoming_police_car: :fire_engine: :ambulance: :minibus: :truck: :train: :station: :train2: :bullettrain_front: :bullettrain_side: :light_rail: :monorail: :railway_car: :trolleybus: :ticket: :fuelpump: :vertical_traffic_light: :traffic_light: :warning: :construction: :beginner: :atm: :slot_machine: :busstop: :barber: :hotsprings: :checkered_flag: :crossed_flags: :izakaya_lantern: :moyai: :circus_tent: :performing_arts: :round_pushpin: :triangular_flag_on_post: :jp: :kr: :cn: :us: :fr: :es: :it: :ru: :gb: :uk: :de: :ca: :eh: :pk: :za:  

webapp/components/suggestion/emoticon_provider.jsx

@@ -55,7 +55,7 @@ export default class EmoticonProvider {
 
             const matched = [];
 
-            for (const [name, emoticon] of Emoticons.emoticons) {
+            for (const [name, emoticon] of Emoticons.getEmoticonsByName()) {
                 if (name.indexOf(partialName) !== -1) {
                     matched.push(emoticon);
 

webapp/package.json

@@ -29,7 +29,7 @@
     "react-router": "2.0.1",
     "react-textarea-autosize": "3.3.0",
     "superagent": "1.8.3",
-    "twemoji": "1.4.1",
+    "twemoji": "2.0.5",
     "velocity-animate": "1.2.3"
   },
   "devDependencies": {

webapp/utils/emoji.json

@@ -8181,6 +8181,14 @@
   , "tags": [
     ]
   }
+, {
+    "aliases": [
+      "mm",
+      "mattermost"
+    ]
+  , "tags": [
+    ]
+  }
 , {
     "aliases": [
       "basecamp"

webapp/utils/emoticons.jsx

@@ -29,10 +29,12 @@ const emoticonPatterns = {
     thumbsdown: /(^|\s)(:\-1:)(?=$|\s)/g // :-1:
 };
 
-export const emoticons = initializeEmoticons();
+let emoticonsByName;
+let emoticonsByCodePoint;
 
 function initializeEmoticons() {
-    const emoticonMap = new Map();
+    emoticonsByName = new Map();
+    emoticonsByCodePoint = new Set();
 
     for (const emoji of emojis) {
         const unicode = emoji.emoji;
@@ -40,6 +42,8 @@ function initializeEmoticons() {
         let filename = '';
         if (unicode) {
             // this is a unicode emoji so the character code determines the file name
+            let codepoint = '';
+
             for (let i = 0; i < unicode.length; i += 2) {
                 const code = fixedCharCodeAt(unicode, i);
 
@@ -50,25 +54,26 @@ function initializeEmoticons() {
 
                 // some emoji (such as country flags) span multiple unicode characters
                 if (i !== 0) {
-                    filename += '-';
+                    codepoint += '-';
                 }
 
-                filename += pad(code.toString(16));
+                codepoint += pad(code.toString(16));
             }
+
+            filename = codepoint;
+            emoticonsByCodePoint.add(codepoint);
         } else {
             // this isn't a unicode emoji so the first alias determines the file name
             filename = emoji.aliases[0];
         }
 
         for (const alias of emoji.aliases) {
-            emoticonMap.set(alias, {
+            emoticonsByName.set(alias, {
                 alias,
                 path: getImagePathForEmoticon(filename)
             });
         }
     }
-
-    return emoticonMap;
 }
 
 // Pads a hexadecimal number with zeroes to be at least 4 digits long
@@ -110,14 +115,30 @@ function fixedCharCodeAt(str, idx = 0) {
     return code;
 }
 
+export function getEmoticonsByName() {
+    if (!emoticonsByName) {
+        initializeEmoticons();
+    }
+
+    return emoticonsByName;
+}
+
+export function getEmoticonsByCodePoint() {
+    if (!emoticonsByCodePoint) {
+        initializeEmoticons();
+    }
+
+    return emoticonsByCodePoint;
+}
+
 export function handleEmoticons(text, tokens) {
     let output = text;
 
     function replaceEmoticonWithToken(fullMatch, prefix, matchText, name) {
-        if (emoticons.has(name)) {
+        if (getEmoticonsByName().has(name)) {
             const index = tokens.size;
             const alias = `MM_EMOTICON${index}`;
-            const path = emoticons.get(name).path;
+            const path = getEmoticonsByName().get(name).path;
 
             tokens.set(alias, {
                 value: `<img align="absmiddle" alt="${matchText}" class="emoticon" src="${path}" title="${matchText}" />`,
@@ -141,6 +162,6 @@ export function handleEmoticons(text, tokens) {
     return output;
 }
 
-function getImagePathForEmoticon(name) {
+export function getImagePathForEmoticon(name) {
     return Constants.EMOJI_PATH + '/' + name + '.png';
 }

webapp/utils/text_formatting.jsx

@@ -60,17 +60,25 @@ export function doFormatText(text, options) {
         output = highlightCurrentMentions(output, tokens);
     }
 
-    // reinsert tokens with formatted versions of the important words and phrases
-    output = replaceTokens(output, tokens);
-
     if (!('emoticons' in options) || options.emoticon) {
         output = twemoji.parse(output, {
             className: 'emoticon',
             base: '',
-            folder: Constants.EMOJI_PATH
+            folder: Constants.EMOJI_PATH,
+            callback: (icon, twemojiOptions) => {
+                if (!Emoticons.getEmoticonsByCodePoint().has(icon)) {
+                    // just leave the unicode characters and hope the browser can handle it
+                    return null;
+                }
+
+                return ''.concat(twemojiOptions.base, twemojiOptions.size, '/', icon, twemojiOptions.ext);
+            }
         });
     }
 
+    // reinsert tokens with formatted versions of the important words and phrases
+    output = replaceTokens(output, tokens);
+
     return output;
 }