[MM-52226] : Fix broken make-emojis command (#23454)

server/public/model/emoji_data.go

@@ -1,6 +1,7 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
-// This file is automatically generated via `make emojis`. Do not modify it manually.
+
+// This file is automatically generated via `/webapp/channels/build/make_emojis.mjs`. Do not modify it manually.
 
 package model
 

webapp/channels/.github/PULL_REQUEST_TEMPLATE.md

@@ -14,7 +14,7 @@ A description of what this pull request does.
 <!--
 If this pull request addresses a Help Wanted ticket, please link the relevant GitHub issue, e.g.
 
-  Fixes https://github.com/mattermost/mattermost-server/issues/XXXXX
+Fixes https://github.com/mattermost/mattermost-server/issues/XXXXX
 
 Otherwise, link the JIRA ticket.
 -->

webapp/channels/Makefile

@@ -1,26 +0,0 @@
-BUILD_SERVER_DIR = ../mattermost-server
-
-export NODE_OPTIONS=--max-old-space-size=4096
-
-.PHONY: i18n-extract
-i18n-extract: ## Extract strings for translation from the source code
-	npm run i18n-extract
-
-.PHONY: emojis
-emojis: ## Creates emoji JSON, JSX and Go files and extracts emoji images from the system font
-	SERVER_DIR=$(BUILD_SERVER_DIR) npm run make-emojis
-	@if [ -e $(BUILD_SERVER_DIR)/model/emoji_data.go ]; then \
-		gofmt -w $(BUILD_SERVER_DIR)/model/emoji_data.go; \
-	fi
-
-## Help documentatin à la https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help
-help:
-	@grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' ./Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
-
-.PHONY: update-dependencies
-update-dependencies: # Updates the dependencies
-	npm update --depth 9999
-	npm audit fix
-	@echo Automatic dependency update complete.
-	@echo You should manually inspect changes to package.json and pin exact versions of packages where appropriate.

webapp/channels/build/emoji/make_emojis.mjs

@@ -2,20 +2,23 @@
 // See LICENSE.txt for license information.
 
 /*
-* This function will generate the emoji files for both the webapp and server to use emojis from emoji-datasource
+* This script will generate the emoji files for both the webapp and server to use emojis from 'emoji-datasource' npm package
 * It will generate the following files:
-* 'mattermost-webapp/utils/emoji.ts'
+* '<rootDir>/webapp/channels/src/utils/emoji.ts'
-* 'mattermost-webapp/sass/components/_emojisprite.scss'
+* '<rootDir>/webapp/channels/src/sass/components/_emojisprite.scss'
-* 'mattermost-webapp/utils/emoji.json'
+* '<rootDir>/webapp/channels/src/utils/emoji.json'
-* 'mattermost-server/model/emoji_data.go', (if server-dir argument is passed with path to server, otherwise it will be generated in './emoji_data.go'")
+* '<rootDir>/server/public/model/emoji_data.go',
 *
 * For help on how to use this script, run:
 * npm run make-emojis -- --help
 */
 
-import path from 'path';
+/* eslint-disable no-console */
-import * as Fs from 'fs/promises';
+
-import {readFileSync} from 'fs';
+import path from 'node:path';
+import * as fsPromise from 'node:fs/promises';
+import * as fs from 'node:fs';
+import * as url from 'node:url';
 
 import yargs from 'yargs';
 import jsonData from 'emoji-datasource/emoji.json';
@@ -28,15 +31,19 @@ const EMOJI_SIZE_PADDED = EMOJI_SIZE + 2; // 1px per side
 const EMOJI_DEFAULT_SKIN = 'default';
 const endResults = [];
 
-const argv = yargs(process.argv.slice(2)).
+const errorLogColor = '\x1b[31m%s\x1b[0m';
+const warnLogColor = '\x1b[33m%s\x1b[0m';
+const successLogColor = '\x1b[32m%s\x1b[0m';
+
+const currentDir = path.dirname(url.fileURLToPath(import.meta.url));
+const rootDir = path.resolve(currentDir, '..', '..', '..', '..');
+const serverRootDir = path.resolve(rootDir, 'server');
+const webappRootDir = path.resolve(rootDir, 'webapp');
+
+const argv = yargs(process.argv.slice(1)).
     scriptName('make-emojis').
     usage('Usage : npm run $0 -- [args]').
     example('npm run $0 -- --excluded-emoji-file ./excludedEmojis.txt', 'removes mentioned emojis from the app').
-    example('npm run $0 -- --server-dir ../mattermost-server', 'path to mattermost-server for copying emoji_data.go file').
-    option('server-dir', {
-        description: 'Path to mattermost-server',
-        type: 'string',
-    }).
     option('excluded-emoji-file', {
         description: 'Path to a file containing emoji short names to exclude',
         type: 'string',
@@ -46,32 +53,44 @@ const argv = yargs(process.argv.slice(2)).
     argv;
 
 const argsExcludedEmojiFile = argv['excluded-emoji-file'];
-const argsServerDirectory = argv['server-dir'];
 
-const successLogColor = '\x1b[32m%s\x1b[0m';
+function writeToFileAndPrint(fileName, filePath, data) {
-const errorLogColor = '\x1b[31m%s\x1b[0m';
+    const promise = fsPromise.writeFile(filePath, data, writeOptions);
-const warnLogColor = '\x1b[33m%s\x1b[0m';
 
-// copy image files
+    promise.then(() => {
-const source = `node_modules/emoji-datasource-apple/img/apple/${EMOJI_SIZE}/`;
+        console.log(successLogColor, `${fileName} generated successfully in ${filePath}\n`);
-const readDirPromise = Fs.readdir(source);
+    });
+    return promise;
+}
+
+function normalizeCategoryName(category) {
+    return category.toLowerCase().replace(' & ', '-');
+}
+
+// Copy emoji images from 'emoji-datasource-apple'
+const emojiDataSourceDir = path.resolve(webappRootDir, `node_modules/emoji-datasource-apple/img/apple/${EMOJI_SIZE}/`);
+const emojiImagesDir = path.resolve(webappRootDir, 'channels', 'src', 'images', 'emoji');
+const readDirPromise = fsPromise.readdir(emojiDataSourceDir);
 endResults.push(readDirPromise);
 readDirPromise.then((images) => {
-    console.log(`Copying ${images.length} emoji images, this might take a while`);
+    console.log(`Copying ${images.length} emoji images, this might take a while\n`);
 
     for (const imageFile of images) {
         endResults.push(
-            Fs.copyFile(path.join(source, imageFile), path.join('images/emoji', imageFile)).
+            fsPromise.copyFile(path.join(emojiDataSourceDir, imageFile), path.join(emojiImagesDir, imageFile)).
                 catch((err) => console.log(errorLogColor, `[ERROR] Failed to copy ${imageFile}: ${err}`)));
     }
 });
-Fs.copyFile('images/icon64x64.png', 'images/emoji/mattermost.png');
 
-// copy sheet image
+const webappImagesDir = path.resolve(webappRootDir, 'channels', 'src', 'images');
-const sheetSource = `node_modules/emoji-datasource-apple/img/apple/sheets/${EMOJI_SIZE}.png`;
+fsPromise.copyFile(path.resolve(webappImagesDir, 'icon64x64.png'), path.resolve(webappImagesDir, 'emoji/mattermost.png'));
+
+const sheetSource = path.resolve(webappRootDir, `node_modules/emoji-datasource-apple/img/apple/sheets/${EMOJI_SIZE}.png`);
+const sheetAbsoluteFile = path.resolve(webappRootDir, 'channels', 'src', 'images/emoji-sheets/apple-sheet.png');
 const sheetFile = 'images/emoji-sheets/apple-sheet.png';
-console.log('Copying sprite sheet');
+
-Fs.copyFile(sheetSource, sheetFile).catch((err) => console.log(errorLogColor, `[ERROR] Failed to copy sheet file: ${err}`));
+// Copy sheet image
+fsPromise.copyFile(sheetSource, sheetAbsoluteFile).catch((err) => console.log(errorLogColor, `[ERROR] Failed to copy sheet file: ${err}`));
 
 // we'll load it as a two dimensional array so we can generate a Map out of it
 const emojiIndicesByAlias = [];
@@ -104,30 +123,19 @@ const writeOptions = {
     signal: control.signal,
 };
 
-function filename(emoji) {
+// Extract excluded emoji shortnames as an array
-    return emoji.image.split('.')[0];
+const excludedEmoji = [];
-}
+if (argsExcludedEmojiFile) {
-
+    fs.readFileSync(path.normalize(argsExcludedEmojiFile), 'utf-8').split(/\r?\n/).forEach((line) => {
-function writeFile(fileName, filePath, data) {
+        excludedEmoji.push(line);
-    const promise = Fs.writeFile(filePath, data, writeOptions);
-
-    promise.then(() => {
-        console.log(successLogColor, `${fileName} generated successfully.`);
     });
-    return promise;
+    console.log(warnLogColor, `\n[WARNING] The following emoji(s) will be excluded from the webapp: \n${excludedEmoji}\n`);
 }
 
-function convertCategory(category) {
+// Remove unwanted emoji
-    return category.toLowerCase().replace(' & ', '-');
+const filteredEmojiJson = jsonData.filter((element) => !excludedEmoji.some((e) => element.short_names.includes(e)));
-}
 
-function addIndexToMap(emojiMap, key, ...indexes) {
+function generateEmojiSkinVariations(emoji) {
-    const newList = emojiMap.get(key) || [];
-    newList.push(...indexes);
-    emojiMap.set(key, newList);
-}
-
-function genSkinVariations(emoji) {
     if (!emoji.skin_variations) {
         return [];
     }
@@ -146,6 +154,79 @@ function genSkinVariations(emoji) {
     });
 }
 
+// populate skin tones as full emojis
+const fullEmoji = [...filteredEmojiJson];
+filteredEmojiJson.forEach((emoji) => {
+    const variations = generateEmojiSkinVariations(emoji);
+    fullEmoji.push(...variations);
+});
+
+// add old shortnames to maintain backwards compatibility with gemoji
+fullEmoji.forEach((emoji) => {
+    if (emoji.short_name in additionalShortnames) {
+        emoji.short_names.push(...additionalShortnames[emoji.short_name]);
+    }
+});
+
+// add built-in custom emojis
+fullEmoji.push({
+    id: 'mattermost',
+    name: 'Mattermost',
+    unified: '',
+    image: 'mattermost.png',
+    short_name: 'mattermost',
+    short_names: ['mattermost'],
+    category: 'custom',
+});
+
+fullEmoji.sort((emojiA, emojiB) => emojiA.sort_order - emojiB.sort_order);
+
+function addIndexToMap(emojiMap, key, ...indexes) {
+    const newList = emojiMap.get(key) || [];
+    newList.push(...indexes);
+    emojiMap.set(key, newList);
+}
+
+const skinset = new Set();
+fullEmoji.forEach((emoji, index) => {
+    if (emoji.unified) {
+        emojiIndicesByUnicode.push([emoji.unified.toLowerCase(), index]);
+    }
+
+    const safeCat = normalizeCategoryName(emoji.category);
+    categoryDefaultTranslation.set(safeCat, emoji.category);
+    emoji.category = safeCat;
+    addIndexToMap(emojiIndicesByCategory, safeCat, index);
+    if (emoji.skins || emoji.skin_variations) {
+        const skin = (emoji.skins && emoji.skins[0]) || EMOJI_DEFAULT_SKIN;
+        skinset.add(skin);
+        const categoryBySkin = emojiIndicesByCategoryAndSkin.get(skin) || new Map();
+        addIndexToMap(categoryBySkin, safeCat, index);
+        emojiIndicesByCategoryAndSkin.set(skin, categoryBySkin);
+    } else {
+        addIndexToMap(emojiIndicesByCategoryNoSkin, safeCat, index);
+    }
+    categoryNamesSet.add(safeCat);
+    emojiIndicesByAlias.push(...emoji.short_names.map((alias) => [alias, index]));
+    const file = emoji.image.split('.')[0];
+    emoji.fileName = emoji.image;
+    emoji.image = file;
+
+    if (emoji.category !== 'custom') {
+        let x = emoji.sheet_x * EMOJI_SIZE_PADDED;
+        if (x !== 0) {
+            x += 'px';
+        }
+        let y = emoji.sheet_y * EMOJI_SIZE_PADDED;
+        if (y !== 0) {
+            y += 'px';
+        }
+        emojiFilePositions.set(file, `-${x} -${y};`);
+    }
+
+    emojiImagesByAlias.push(...emoji.short_names.map((alias) => `"${alias}": "${file}"`));
+});
+
 function trimPropertiesFromEmoji(emoji) {
     if (emoji.hasOwnProperty('non_qualified')) {
         Reflect.deleteProperty(emoji, 'non_qualified');
@@ -218,92 +299,14 @@ function trimPropertiesFromEmoji(emoji) {
     return emoji;
 }
 
-// Extract excluded emoji shortnames as an array
-const excludedEmoji = [];
-if (argsExcludedEmojiFile) {
-    readFileSync(path.normalize(argsExcludedEmojiFile), 'utf-8').split(/\r?\n/).forEach((line) => {
-        excludedEmoji.push(line);
-    });
-    console.log(warnLogColor, `\n[WARNING] The following emoji(s) will be excluded from the webapp: \n${excludedEmoji}\n`);
-}
-
-// Remove unwanted emoji
-const filteredEmojiJson = jsonData.filter((element) => !excludedEmoji.some((e) => element.short_names.includes(e)));
-
-// populate skin tones as full emojis
-const fullEmoji = [...filteredEmojiJson];
-filteredEmojiJson.forEach((emoji) => {
-    const variations = genSkinVariations(emoji);
-    fullEmoji.push(...variations);
-});
-
-// add old shortnames to maintain backwards compatibility with gemoji
-fullEmoji.forEach((emoji) => {
-    if (emoji.short_name in additionalShortnames) {
-        emoji.short_names.push(...additionalShortnames[emoji.short_name]);
-    }
-});
-
-// add built-in custom emojis
-fullEmoji.push({
-    id: 'mattermost',
-    name: 'Mattermost',
-    unified: '',
-    image: 'mattermost.png',
-    short_name: 'mattermost',
-    short_names: ['mattermost'],
-    category: 'custom',
-});
-
-fullEmoji.sort((emojiA, emojiB) => emojiA.sort_order - emojiB.sort_order);
-
-const skinset = new Set();
-fullEmoji.forEach((emoji, index) => {
-    if (emoji.unified) {
-        emojiIndicesByUnicode.push([emoji.unified.toLowerCase(), index]);
-    }
-
-    const safeCat = convertCategory(emoji.category);
-    categoryDefaultTranslation.set(safeCat, emoji.category);
-    emoji.category = safeCat;
-    addIndexToMap(emojiIndicesByCategory, safeCat, index);
-    if (emoji.skins || emoji.skin_variations) {
-        const skin = (emoji.skins && emoji.skins[0]) || EMOJI_DEFAULT_SKIN;
-        skinset.add(skin);
-        const categoryBySkin = emojiIndicesByCategoryAndSkin.get(skin) || new Map();
-        addIndexToMap(categoryBySkin, safeCat, index);
-        emojiIndicesByCategoryAndSkin.set(skin, categoryBySkin);
-    } else {
-        addIndexToMap(emojiIndicesByCategoryNoSkin, safeCat, index);
-    }
-    categoryNamesSet.add(safeCat);
-    emojiIndicesByAlias.push(...emoji.short_names.map((alias) => [alias, index]));
-    const file = filename(emoji);
-    emoji.fileName = emoji.image;
-    emoji.image = file;
-
-    if (emoji.category !== 'custom') {
-        let x = emoji.sheet_x * EMOJI_SIZE_PADDED;
-        if (x !== 0) {
-            x += 'px';
-        }
-        let y = emoji.sheet_y * EMOJI_SIZE_PADDED;
-        if (y !== 0) {
-            y += 'px';
-        }
-        emojiFilePositions.set(file, `-${x} -${y};`);
-    }
-
-    emojiImagesByAlias.push(...emoji.short_names.map((alias) => `"${alias}": "${file}"`));
-});
-
 // Removed properties that are not needed for the webapp
 const trimmedDownEmojis = fullEmoji.map((emoji) => trimPropertiesFromEmoji(emoji));
 
-// write emoji.json
+// Create the emoji.json file
-endResults.push(writeFile('emoji.json', 'utils/emoji.json', JSON.stringify(trimmedDownEmojis, null, 4)));
+const webappUtilsEmojiDir = path.resolve(webappRootDir, 'channels', 'src', 'utils', 'emoji.json');
+endResults.push(writeToFileAndPrint('emoji.json', webappUtilsEmojiDir, JSON.stringify(trimmedDownEmojis, null, 4)));
 
-const categoryList = Object.keys(jsonCategories).filter((item) => item !== 'Component').map(convertCategory);
+const categoryList = Object.keys(jsonCategories).filter((item) => item !== 'Component').map(normalizeCategoryName);
 const categoryNames = ['recent', ...categoryList, 'custom'];
 categoryDefaultTranslation.set('recent', 'Recently Used');
 categoryDefaultTranslation.set('searchResults', 'Search Results');
@@ -319,8 +322,13 @@ for (const skin of emojiIndicesByCategoryAndSkin.keys()) {
     skinnedCats.push(`['${skin}', genSkinnedCategories('${skin}')]`);
 }
 
-// generate emoji.ts out of the emoji.json parsing we did
+// Generate contents of emoji.ts out of the emoji.json parsing we did earlier
-const emojiJSX = `// This file is automatically generated via \`make emojis\`. Do not modify it manually.
+const emojiJSX = `// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// This file is automatically generated via \`/webapp/channels/build/make_emojis.mjs\`. Do not modify it manually.
+
+/* eslint-disable */
 
 import {t} from 'utils/i18n';
 
@@ -371,14 +379,15 @@ const getSkinnedCategories = memoize(genSkinnedCategories);
 export const EmojiIndicesByCategory = new Map([${skinnedCats.join(', ')}]);
 `;
 
-// write emoji.ts
+// Create the emoji.ts file
-endResults.push(writeFile('emoji.ts', 'utils/emoji.ts', emojiJSX));
+const webappUtilsEmojiTSDir = path.resolve(webappRootDir, 'channels', 'src', 'utils', 'emoji.ts');
-
+endResults.push(writeToFileAndPrint('emoji.ts', webappUtilsEmojiTSDir, emojiJSX));
-// golang emoji data
 
+const serverEmojiDataDir = path.resolve(serverRootDir, 'public', 'model', 'emoji_data.go');
 const emojiGo = `// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
-// This file is automatically generated via \`make emojis\`. Do not modify it manually.
+
+// This file is automatically generated via \`/webapp/channels/build/make_emojis.mjs\`. Do not modify it manually.
 
 package model
 
@@ -388,26 +397,11 @@ ${emojiImagesByAlias.join(`,
 }
 `;
 
-const goPromise = writeFile('emoji_data.go', 'emoji_data.go', emojiGo);
+// Create the emoji_data.go file
+const goPromise = writeToFileAndPrint('emoji_data.go', serverEmojiDataDir, emojiGo);
 endResults.push(goPromise);
 
-// If server-dir is defined we can update the file emoji_data.go in the server directory
+// Create individual emoji styles
-if (argsServerDirectory) {
-    const destination = path.join(argsServerDirectory, 'model/emoji_data.go');
-    goPromise.then(() => {
-        // this is an obvious race condition, as goPromise might be the last one, and then executed out of the `all` call below,
-        // but it shouldn't be any problem other than a log out of place and a need to do an explicit catch.
-        const mvPromise = Fs.rename('emoji_data.go', destination);
-        endResults.push(mvPromise);
-        mvPromise.catch((err) => {
-            console.log(errorLogColor, `[ERROR] There was an error trying to move the emoji_data.go file: ${err}`);
-        });
-    });
-} else {
-    console.log(warnLogColor, '\n[WARNING] server-dir path not defined, `emoji_data.go` will be located in the root of this project, remember to move it to the server\n');
-}
-
-// sprite css file
 const cssCats = categoryNames.filter((cat) => cat !== 'custom').map((cat) => `.emoji-category-${cat} { background-image: url('${sheetFile}'); }`);
 const cssEmojis = [];
 for (const key of emojiFilePositions.keys()) {
@@ -456,12 +450,15 @@ ${cssCats.join('\n')}
 ${cssEmojis.join('\n')}
 `;
 
-// write emoji.ts
+// Create the emojisprite.scss file
-endResults.push(writeFile('_emojisprite.scss', 'sass/components/_emojisprite.scss', cssRules));
+const emojispriteDir = path.resolve(webappRootDir, 'channels', 'src', 'sass', 'components', '_emojisprite.scss');
+endResults.push(writeToFileAndPrint('_emojisprite.scss', emojispriteDir, cssRules));
 
 Promise.all(endResults).then(() => {
-    console.log(warnLogColor, '\n[WARNING] Remember to run `make i18n-extract` as categories might have changed.');
+    console.log(warnLogColor, '\n[WARNING] Remember to npm run \'i18n-extract\' as categories might have changed.');
+    console.log(warnLogColor, '[WARNING] Remember to run `gofmt -w ./server/public/model/emoji_data.go` from the root of the repository.');
+    console.log(successLogColor, '\n[SUCCESS] make-emojis script completed successfully.');
 }).catch((err) => {
     control.abort(); // cancel any other file writing
-    console.log(errorLogColor, `[ERROR] There was an error writing emojis: ${err}`);
+    console.log(errorLogColor, `[ERROR] There was an error while running make-emojis script: ${err}`);
 });

webapp/channels/package.json

@@ -217,6 +217,6 @@
     "i18n-clean-empty": "npm run mmjstool -- i18n clean-empty --webapp-dir ./src",
     "i18n-check-empty-src": "npm run mmjstool -- i18n check-empty-src --webapp-dir ./src",
     "check-types": "tsc -b",
-    "make-emojis": "node --experimental-json-modules build/emoji/emoji_gen.mjs"
+    "make-emojis": "node --experimental-json-modules build/emoji/make_emojis.mjs"
   }
 }

webapp/channels/src/utils/emoji.ts

@@ -1,4 +1,7 @@
-// This file is automatically generated via `make emojis`. Do not modify it manually.
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// This file is automatically generated via `/webapp/channels/build/make_emojis.mjs`. Do not modify it manually.
 
 /* eslint-disable */