[MM-52226] : Fix broken make-emojis command (#23454)

This commit is contained in:
M-ZubairAhmed 2023-05-26 10:43:44 +05:30 committed by GitHub
parent 69196b2986
commit b72043727a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 164 additions and 189 deletions

View File

@ -1,6 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// This file is automatically generated via `make emojis`. Do not modify it manually.
// This file is automatically generated via `/webapp/channels/build/make_emojis.mjs`. Do not modify it manually.
package model

View File

@ -14,7 +14,7 @@ A description of what this pull request does.
<!--
If this pull request addresses a Help Wanted ticket, please link the relevant GitHub issue, e.g.
Fixes https://github.com/mattermost/mattermost-server/issues/XXXXX
Fixes https://github.com/mattermost/mattermost-server/issues/XXXXX
Otherwise, link the JIRA ticket.
-->

View File

@ -1,26 +0,0 @@
BUILD_SERVER_DIR = ../mattermost-server
export NODE_OPTIONS=--max-old-space-size=4096
.PHONY: i18n-extract
i18n-extract: ## Extract strings for translation from the source code
npm run i18n-extract
.PHONY: emojis
emojis: ## Creates emoji JSON, JSX and Go files and extracts emoji images from the system font
SERVER_DIR=$(BUILD_SERVER_DIR) npm run make-emojis
@if [ -e $(BUILD_SERVER_DIR)/model/emoji_data.go ]; then \
gofmt -w $(BUILD_SERVER_DIR)/model/emoji_data.go; \
fi
## Help documentatin à la https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help:
@grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' ./Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
.PHONY: update-dependencies
update-dependencies: # Updates the dependencies
npm update --depth 9999
npm audit fix
@echo Automatic dependency update complete.
@echo You should manually inspect changes to package.json and pin exact versions of packages where appropriate.

View File

@ -2,20 +2,23 @@
// See LICENSE.txt for license information.
/*
* This function will generate the emoji files for both the webapp and server to use emojis from emoji-datasource
* This script will generate the emoji files for both the webapp and server to use emojis from 'emoji-datasource' npm package
* It will generate the following files:
* 'mattermost-webapp/utils/emoji.ts'
* 'mattermost-webapp/sass/components/_emojisprite.scss'
* 'mattermost-webapp/utils/emoji.json'
* 'mattermost-server/model/emoji_data.go', (if server-dir argument is passed with path to server, otherwise it will be generated in './emoji_data.go'")
* '<rootDir>/webapp/channels/src/utils/emoji.ts'
* '<rootDir>/webapp/channels/src/sass/components/_emojisprite.scss'
* '<rootDir>/webapp/channels/src/utils/emoji.json'
* '<rootDir>/server/public/model/emoji_data.go',
*
* For help on how to use this script, run:
* npm run make-emojis -- --help
*/
import path from 'path';
import * as Fs from 'fs/promises';
import {readFileSync} from 'fs';
/* eslint-disable no-console */
import path from 'node:path';
import * as fsPromise from 'node:fs/promises';
import * as fs from 'node:fs';
import * as url from 'node:url';
import yargs from 'yargs';
import jsonData from 'emoji-datasource/emoji.json';
@ -28,15 +31,19 @@ const EMOJI_SIZE_PADDED = EMOJI_SIZE + 2; // 1px per side
const EMOJI_DEFAULT_SKIN = 'default';
const endResults = [];
const argv = yargs(process.argv.slice(2)).
const errorLogColor = '\x1b[31m%s\x1b[0m';
const warnLogColor = '\x1b[33m%s\x1b[0m';
const successLogColor = '\x1b[32m%s\x1b[0m';
const currentDir = path.dirname(url.fileURLToPath(import.meta.url));
const rootDir = path.resolve(currentDir, '..', '..', '..', '..');
const serverRootDir = path.resolve(rootDir, 'server');
const webappRootDir = path.resolve(rootDir, 'webapp');
const argv = yargs(process.argv.slice(1)).
scriptName('make-emojis').
usage('Usage : npm run $0 -- [args]').
example('npm run $0 -- --excluded-emoji-file ./excludedEmojis.txt', 'removes mentioned emojis from the app').
example('npm run $0 -- --server-dir ../mattermost-server', 'path to mattermost-server for copying emoji_data.go file').
option('server-dir', {
description: 'Path to mattermost-server',
type: 'string',
}).
option('excluded-emoji-file', {
description: 'Path to a file containing emoji short names to exclude',
type: 'string',
@ -46,32 +53,44 @@ const argv = yargs(process.argv.slice(2)).
argv;
const argsExcludedEmojiFile = argv['excluded-emoji-file'];
const argsServerDirectory = argv['server-dir'];
const successLogColor = '\x1b[32m%s\x1b[0m';
const errorLogColor = '\x1b[31m%s\x1b[0m';
const warnLogColor = '\x1b[33m%s\x1b[0m';
function writeToFileAndPrint(fileName, filePath, data) {
const promise = fsPromise.writeFile(filePath, data, writeOptions);
// copy image files
const source = `node_modules/emoji-datasource-apple/img/apple/${EMOJI_SIZE}/`;
const readDirPromise = Fs.readdir(source);
promise.then(() => {
console.log(successLogColor, `${fileName} generated successfully in ${filePath}\n`);
});
return promise;
}
function normalizeCategoryName(category) {
return category.toLowerCase().replace(' & ', '-');
}
// Copy emoji images from 'emoji-datasource-apple'
const emojiDataSourceDir = path.resolve(webappRootDir, `node_modules/emoji-datasource-apple/img/apple/${EMOJI_SIZE}/`);
const emojiImagesDir = path.resolve(webappRootDir, 'channels', 'src', 'images', 'emoji');
const readDirPromise = fsPromise.readdir(emojiDataSourceDir);
endResults.push(readDirPromise);
readDirPromise.then((images) => {
console.log(`Copying ${images.length} emoji images, this might take a while`);
console.log(`Copying ${images.length} emoji images, this might take a while\n`);
for (const imageFile of images) {
endResults.push(
Fs.copyFile(path.join(source, imageFile), path.join('images/emoji', imageFile)).
fsPromise.copyFile(path.join(emojiDataSourceDir, imageFile), path.join(emojiImagesDir, imageFile)).
catch((err) => console.log(errorLogColor, `[ERROR] Failed to copy ${imageFile}: ${err}`)));
}
});
Fs.copyFile('images/icon64x64.png', 'images/emoji/mattermost.png');
// copy sheet image
const sheetSource = `node_modules/emoji-datasource-apple/img/apple/sheets/${EMOJI_SIZE}.png`;
const webappImagesDir = path.resolve(webappRootDir, 'channels', 'src', 'images');
fsPromise.copyFile(path.resolve(webappImagesDir, 'icon64x64.png'), path.resolve(webappImagesDir, 'emoji/mattermost.png'));
const sheetSource = path.resolve(webappRootDir, `node_modules/emoji-datasource-apple/img/apple/sheets/${EMOJI_SIZE}.png`);
const sheetAbsoluteFile = path.resolve(webappRootDir, 'channels', 'src', 'images/emoji-sheets/apple-sheet.png');
const sheetFile = 'images/emoji-sheets/apple-sheet.png';
console.log('Copying sprite sheet');
Fs.copyFile(sheetSource, sheetFile).catch((err) => console.log(errorLogColor, `[ERROR] Failed to copy sheet file: ${err}`));
// Copy sheet image
fsPromise.copyFile(sheetSource, sheetAbsoluteFile).catch((err) => console.log(errorLogColor, `[ERROR] Failed to copy sheet file: ${err}`));
// we'll load it as a two dimensional array so we can generate a Map out of it
const emojiIndicesByAlias = [];
@ -104,30 +123,19 @@ const writeOptions = {
signal: control.signal,
};
function filename(emoji) {
return emoji.image.split('.')[0];
}
function writeFile(fileName, filePath, data) {
const promise = Fs.writeFile(filePath, data, writeOptions);
promise.then(() => {
console.log(successLogColor, `${fileName} generated successfully.`);
// Extract excluded emoji shortnames as an array
const excludedEmoji = [];
if (argsExcludedEmojiFile) {
fs.readFileSync(path.normalize(argsExcludedEmojiFile), 'utf-8').split(/\r?\n/).forEach((line) => {
excludedEmoji.push(line);
});
return promise;
console.log(warnLogColor, `\n[WARNING] The following emoji(s) will be excluded from the webapp: \n${excludedEmoji}\n`);
}
function convertCategory(category) {
return category.toLowerCase().replace(' & ', '-');
}
// Remove unwanted emoji
const filteredEmojiJson = jsonData.filter((element) => !excludedEmoji.some((e) => element.short_names.includes(e)));
function addIndexToMap(emojiMap, key, ...indexes) {
const newList = emojiMap.get(key) || [];
newList.push(...indexes);
emojiMap.set(key, newList);
}
function genSkinVariations(emoji) {
function generateEmojiSkinVariations(emoji) {
if (!emoji.skin_variations) {
return [];
}
@ -146,6 +154,79 @@ function genSkinVariations(emoji) {
});
}
// populate skin tones as full emojis
const fullEmoji = [...filteredEmojiJson];
filteredEmojiJson.forEach((emoji) => {
const variations = generateEmojiSkinVariations(emoji);
fullEmoji.push(...variations);
});
// add old shortnames to maintain backwards compatibility with gemoji
fullEmoji.forEach((emoji) => {
if (emoji.short_name in additionalShortnames) {
emoji.short_names.push(...additionalShortnames[emoji.short_name]);
}
});
// add built-in custom emojis
fullEmoji.push({
id: 'mattermost',
name: 'Mattermost',
unified: '',
image: 'mattermost.png',
short_name: 'mattermost',
short_names: ['mattermost'],
category: 'custom',
});
fullEmoji.sort((emojiA, emojiB) => emojiA.sort_order - emojiB.sort_order);
function addIndexToMap(emojiMap, key, ...indexes) {
const newList = emojiMap.get(key) || [];
newList.push(...indexes);
emojiMap.set(key, newList);
}
const skinset = new Set();
fullEmoji.forEach((emoji, index) => {
if (emoji.unified) {
emojiIndicesByUnicode.push([emoji.unified.toLowerCase(), index]);
}
const safeCat = normalizeCategoryName(emoji.category);
categoryDefaultTranslation.set(safeCat, emoji.category);
emoji.category = safeCat;
addIndexToMap(emojiIndicesByCategory, safeCat, index);
if (emoji.skins || emoji.skin_variations) {
const skin = (emoji.skins && emoji.skins[0]) || EMOJI_DEFAULT_SKIN;
skinset.add(skin);
const categoryBySkin = emojiIndicesByCategoryAndSkin.get(skin) || new Map();
addIndexToMap(categoryBySkin, safeCat, index);
emojiIndicesByCategoryAndSkin.set(skin, categoryBySkin);
} else {
addIndexToMap(emojiIndicesByCategoryNoSkin, safeCat, index);
}
categoryNamesSet.add(safeCat);
emojiIndicesByAlias.push(...emoji.short_names.map((alias) => [alias, index]));
const file = emoji.image.split('.')[0];
emoji.fileName = emoji.image;
emoji.image = file;
if (emoji.category !== 'custom') {
let x = emoji.sheet_x * EMOJI_SIZE_PADDED;
if (x !== 0) {
x += 'px';
}
let y = emoji.sheet_y * EMOJI_SIZE_PADDED;
if (y !== 0) {
y += 'px';
}
emojiFilePositions.set(file, `-${x} -${y};`);
}
emojiImagesByAlias.push(...emoji.short_names.map((alias) => `"${alias}": "${file}"`));
});
function trimPropertiesFromEmoji(emoji) {
if (emoji.hasOwnProperty('non_qualified')) {
Reflect.deleteProperty(emoji, 'non_qualified');
@ -218,92 +299,14 @@ function trimPropertiesFromEmoji(emoji) {
return emoji;
}
// Extract excluded emoji shortnames as an array
const excludedEmoji = [];
if (argsExcludedEmojiFile) {
readFileSync(path.normalize(argsExcludedEmojiFile), 'utf-8').split(/\r?\n/).forEach((line) => {
excludedEmoji.push(line);
});
console.log(warnLogColor, `\n[WARNING] The following emoji(s) will be excluded from the webapp: \n${excludedEmoji}\n`);
}
// Remove unwanted emoji
const filteredEmojiJson = jsonData.filter((element) => !excludedEmoji.some((e) => element.short_names.includes(e)));
// populate skin tones as full emojis
const fullEmoji = [...filteredEmojiJson];
filteredEmojiJson.forEach((emoji) => {
const variations = genSkinVariations(emoji);
fullEmoji.push(...variations);
});
// add old shortnames to maintain backwards compatibility with gemoji
fullEmoji.forEach((emoji) => {
if (emoji.short_name in additionalShortnames) {
emoji.short_names.push(...additionalShortnames[emoji.short_name]);
}
});
// add built-in custom emojis
fullEmoji.push({
id: 'mattermost',
name: 'Mattermost',
unified: '',
image: 'mattermost.png',
short_name: 'mattermost',
short_names: ['mattermost'],
category: 'custom',
});
fullEmoji.sort((emojiA, emojiB) => emojiA.sort_order - emojiB.sort_order);
const skinset = new Set();
fullEmoji.forEach((emoji, index) => {
if (emoji.unified) {
emojiIndicesByUnicode.push([emoji.unified.toLowerCase(), index]);
}
const safeCat = convertCategory(emoji.category);
categoryDefaultTranslation.set(safeCat, emoji.category);
emoji.category = safeCat;
addIndexToMap(emojiIndicesByCategory, safeCat, index);
if (emoji.skins || emoji.skin_variations) {
const skin = (emoji.skins && emoji.skins[0]) || EMOJI_DEFAULT_SKIN;
skinset.add(skin);
const categoryBySkin = emojiIndicesByCategoryAndSkin.get(skin) || new Map();
addIndexToMap(categoryBySkin, safeCat, index);
emojiIndicesByCategoryAndSkin.set(skin, categoryBySkin);
} else {
addIndexToMap(emojiIndicesByCategoryNoSkin, safeCat, index);
}
categoryNamesSet.add(safeCat);
emojiIndicesByAlias.push(...emoji.short_names.map((alias) => [alias, index]));
const file = filename(emoji);
emoji.fileName = emoji.image;
emoji.image = file;
if (emoji.category !== 'custom') {
let x = emoji.sheet_x * EMOJI_SIZE_PADDED;
if (x !== 0) {
x += 'px';
}
let y = emoji.sheet_y * EMOJI_SIZE_PADDED;
if (y !== 0) {
y += 'px';
}
emojiFilePositions.set(file, `-${x} -${y};`);
}
emojiImagesByAlias.push(...emoji.short_names.map((alias) => `"${alias}": "${file}"`));
});
// Removed properties that are not needed for the webapp
const trimmedDownEmojis = fullEmoji.map((emoji) => trimPropertiesFromEmoji(emoji));
// write emoji.json
endResults.push(writeFile('emoji.json', 'utils/emoji.json', JSON.stringify(trimmedDownEmojis, null, 4)));
// Create the emoji.json file
const webappUtilsEmojiDir = path.resolve(webappRootDir, 'channels', 'src', 'utils', 'emoji.json');
endResults.push(writeToFileAndPrint('emoji.json', webappUtilsEmojiDir, JSON.stringify(trimmedDownEmojis, null, 4)));
const categoryList = Object.keys(jsonCategories).filter((item) => item !== 'Component').map(convertCategory);
const categoryList = Object.keys(jsonCategories).filter((item) => item !== 'Component').map(normalizeCategoryName);
const categoryNames = ['recent', ...categoryList, 'custom'];
categoryDefaultTranslation.set('recent', 'Recently Used');
categoryDefaultTranslation.set('searchResults', 'Search Results');
@ -319,8 +322,13 @@ for (const skin of emojiIndicesByCategoryAndSkin.keys()) {
skinnedCats.push(`['${skin}', genSkinnedCategories('${skin}')]`);
}
// generate emoji.ts out of the emoji.json parsing we did
const emojiJSX = `// This file is automatically generated via \`make emojis\`. Do not modify it manually.
// Generate contents of emoji.ts out of the emoji.json parsing we did earlier
const emojiJSX = `// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// This file is automatically generated via \`/webapp/channels/build/make_emojis.mjs\`. Do not modify it manually.
/* eslint-disable */
import {t} from 'utils/i18n';
@ -371,14 +379,15 @@ const getSkinnedCategories = memoize(genSkinnedCategories);
export const EmojiIndicesByCategory = new Map([${skinnedCats.join(', ')}]);
`;
// write emoji.ts
endResults.push(writeFile('emoji.ts', 'utils/emoji.ts', emojiJSX));
// golang emoji data
// Create the emoji.ts file
const webappUtilsEmojiTSDir = path.resolve(webappRootDir, 'channels', 'src', 'utils', 'emoji.ts');
endResults.push(writeToFileAndPrint('emoji.ts', webappUtilsEmojiTSDir, emojiJSX));
const serverEmojiDataDir = path.resolve(serverRootDir, 'public', 'model', 'emoji_data.go');
const emojiGo = `// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// This file is automatically generated via \`make emojis\`. Do not modify it manually.
// This file is automatically generated via \`/webapp/channels/build/make_emojis.mjs\`. Do not modify it manually.
package model
@ -388,26 +397,11 @@ ${emojiImagesByAlias.join(`,
}
`;
const goPromise = writeFile('emoji_data.go', 'emoji_data.go', emojiGo);
// Create the emoji_data.go file
const goPromise = writeToFileAndPrint('emoji_data.go', serverEmojiDataDir, emojiGo);
endResults.push(goPromise);
// If server-dir is defined we can update the file emoji_data.go in the server directory
if (argsServerDirectory) {
const destination = path.join(argsServerDirectory, 'model/emoji_data.go');
goPromise.then(() => {
// this is an obvious race condition, as goPromise might be the last one, and then executed out of the `all` call below,
// but it shouldn't be any problem other than a log out of place and a need to do an explicit catch.
const mvPromise = Fs.rename('emoji_data.go', destination);
endResults.push(mvPromise);
mvPromise.catch((err) => {
console.log(errorLogColor, `[ERROR] There was an error trying to move the emoji_data.go file: ${err}`);
});
});
} else {
console.log(warnLogColor, '\n[WARNING] server-dir path not defined, `emoji_data.go` will be located in the root of this project, remember to move it to the server\n');
}
// sprite css file
// Create individual emoji styles
const cssCats = categoryNames.filter((cat) => cat !== 'custom').map((cat) => `.emoji-category-${cat} { background-image: url('${sheetFile}'); }`);
const cssEmojis = [];
for (const key of emojiFilePositions.keys()) {
@ -456,12 +450,15 @@ ${cssCats.join('\n')}
${cssEmojis.join('\n')}
`;
// write emoji.ts
endResults.push(writeFile('_emojisprite.scss', 'sass/components/_emojisprite.scss', cssRules));
// Create the emojisprite.scss file
const emojispriteDir = path.resolve(webappRootDir, 'channels', 'src', 'sass', 'components', '_emojisprite.scss');
endResults.push(writeToFileAndPrint('_emojisprite.scss', emojispriteDir, cssRules));
Promise.all(endResults).then(() => {
console.log(warnLogColor, '\n[WARNING] Remember to run `make i18n-extract` as categories might have changed.');
console.log(warnLogColor, '\n[WARNING] Remember to npm run \'i18n-extract\' as categories might have changed.');
console.log(warnLogColor, '[WARNING] Remember to run `gofmt -w ./server/public/model/emoji_data.go` from the root of the repository.');
console.log(successLogColor, '\n[SUCCESS] make-emojis script completed successfully.');
}).catch((err) => {
control.abort(); // cancel any other file writing
console.log(errorLogColor, `[ERROR] There was an error writing emojis: ${err}`);
console.log(errorLogColor, `[ERROR] There was an error while running make-emojis script: ${err}`);
});

View File

@ -217,6 +217,6 @@
"i18n-clean-empty": "npm run mmjstool -- i18n clean-empty --webapp-dir ./src",
"i18n-check-empty-src": "npm run mmjstool -- i18n check-empty-src --webapp-dir ./src",
"check-types": "tsc -b",
"make-emojis": "node --experimental-json-modules build/emoji/emoji_gen.mjs"
"make-emojis": "node --experimental-json-modules build/emoji/make_emojis.mjs"
}
}

View File

@ -1,4 +1,7 @@
// This file is automatically generated via `make emojis`. Do not modify it manually.
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// This file is automatically generated via `/webapp/channels/build/make_emojis.mjs`. Do not modify it manually.
/* eslint-disable */