I18n: Support for Enterprise translations (#86215)

* I18n: Support for Enterprise translations

* don't attempt to link to enterprise in tests

* move extract script to makefile to optionally support enterprise

* update references to old extract script

* update docs

* thank god for unit tests

.drone.yml

@@ -243,11 +243,11 @@ steps:
 - commands:
   - apk add --update git
   - |-
-    yarn run i18n:extract || (echo "
+    make i18n-extract || (echo "
     Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false)
   - "\n            file_diff=$(git diff --dirstat public/locales)\n            if
     [ -n \"$file_diff\" ]; then\n                echo $file_diff\n                echo
-    \"\nTranslation extraction has not been committed. Please run 'yarn i18n:extract',
+    \"\nTranslation extraction has not been committed. Please run 'make i18n-extract',
     commit the changes and push again.\"\n                exit 1\n            fi\n
     \           "
   depends_on:
@@ -1627,11 +1627,11 @@ steps:
 - commands:
   - apk add --update git
   - |-
-    yarn run i18n:extract || (echo "
+    make i18n-extract || (echo "
     Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false)
   - "\n            file_diff=$(git diff --dirstat public/locales)\n            if
     [ -n \"$file_diff\" ]; then\n                echo $file_diff\n                echo
-    \"\nTranslation extraction has not been committed. Please run 'yarn i18n:extract',
+    \"\nTranslation extraction has not been committed. Please run 'make i18n-extract',
     commit the changes and push again.\"\n                exit 1\n            fi\n
     \           "
   depends_on:
@@ -4925,6 +4925,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: fbd59890dac44eb6fb34562f2b2b2db0fc0a8a50d451f9887f815ee35757e0f6
+hmac: 958ed40ca0620498c01fa867b05a1d6c3bcd908067fbb34be691923e95cfc77b
 
 ...

.prettierignore

@@ -19,10 +19,6 @@ vendor
 # TS generate from cue by cuetsy
 **/*.gen.ts
 
-# Auto-generated internationalization files
-public/locales/_build/
-public/locales/**/*.js
-
 # Auto-generated theme files
 theme.light.generated.json
 theme.dark.generated.json

Makefile

@@ -110,6 +110,25 @@ OAPI_SPEC_TARGET = public/openapi3.json
 openapi3-gen: swagger-gen ## Generates OpenApi 3 specs from the Swagger 2 already generated
 	$(GO) run scripts/openapi3/openapi3conv.go $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET)
 
+##@ Internationalisation
+.PHONY: i18n-extract-enterprise
+ENTERPRISE_FE_EXT_FILE = public/app/extensions/index.ts
+ifeq ("$(wildcard $(ENTERPRISE_FE_EXT_FILE))","") ## if enterprise is not enabled
+i18n-extract-enterprise:
+	@echo "Skipping i18n extract for Enterprise: not enabled"
+else
+i18n-extract-enterprise: $(SWAGGER) ## Generate API Swagger specification
+	@echo "Extracting i18n strings for Enterprise"
+	yarn run i18next --config public/locales/i18next-parser-enterprise.config.cjs
+	node ./public/locales/pseudo.mjs --mode enterprise
+endif
+
+.PHONY: i18n-extract
+i18n-extract: i18n-extract-enterprise
+	@echo "Extracting i18n strings for OSS"
+	yarn run i18next --config public/locales/i18next-parser.config.cjs
+	node ./public/locales/pseudo.mjs --mode oss
+
 ##@ Building
 .PHONY: gen-cue
 gen-cue: ## Do all CUE/Thema code generation

contribute/internationalization.md

@@ -9,7 +9,7 @@ Grafana uses the [i18next](https://www.i18next.com/) framework for managing tran
 - Use `<Trans i18nKey="search-results.panel-link">Go to {{ pageTitle }}</Trans>` in code to add a translatable phrase
 - Translations are stored in JSON files in `public/locales/{locale}/grafana.json`
 - If a particular phrase is not available in the a language then it will fall back to English
-- To update phrases in English, edit the default phrase in the component's source and then run `yarn i18n:extract`.
+- To update phrases in English, edit the default phrase in the component's source and then run `make i18n-extract`.
 - The single source of truth for en-US (fallback language) is in grafana/grafana, the single source of truth for any translated language is Crowdin
 - To update phrases in any translated language, edit the phrase in Crowdin. Do not edit the `{locale}/grafana.json`
 
@@ -41,7 +41,7 @@ const ErrorMessage = ({ id, message }) => <Trans i18nKey={`errors.${id}`}>There
 
 2. Upon reload, the default English phrase will appear on the page.
 
-3. Before submitting your PR, run the `yarn i18n:extract` command to extract the messages you added into the `public/locales/en-US/grafana.json` file and make them available for translation.
+3. Before submitting your PR, run the `make i18n-extract` command to extract the messages you added into the `public/locales/en-US/grafana.json` file and make them available for translation.
    **Note:** All other languages will receive their translations when they are ready to be downloaded from Crowdin.
 
 ### Plain JS usage
@@ -79,6 +79,7 @@ While the `t` function can technically be used outside of React functions (e.g,
    1. Add a new constant for the new language
    2. Add the new constant to the `LOCALES` array
    3. Create a PR with the changes and merge when you are ready to release the new language (probably wait until we have translations for it)
+4. In the Enterprise repo, update `src/public/locales/localeExtensions.ts`
 
 ## How translations work in Grafana
 
@@ -186,7 +187,7 @@ import { t } from 'app/core/internationalization';
 const translatedString = t('inbox.heading', 'You got {{count}} message', { count: messages.length });
 ```
 
-Once extracted with `yarn i18n:extract` you will need to manually edit the [English grafana.json message catalogue](../public/locales/en-US/grafana.json) to correct the plural forms. See the [react-i18next docs](https://react.i18next.com/latest/trans-component#plural) for more details.
+Once extracted with `make i18n-extract` you will need to manually edit the [English grafana.json message catalogue](../public/locales/en-US/grafana.json) to correct the plural forms. See the [react-i18next docs](https://react.i18next.com/latest/trans-component#plural) for more details.
 
 ```json
 {

package.json

@@ -48,9 +48,6 @@
     "plugins:build-bundled": "find plugins-bundled -name package.json -not -path '*/node_modules/*' -execdir yarn build \\;",
     "watch": "yarn start -d watch,start core:start --watchTheme",
     "ci:test-frontend": "yarn run test:ci",
-    "i18n:clean": "rimraf public/locales/en-US/grafana.json",
-    "i18n:extract": "yarn run i18next -c public/locales/i18next-parser.config.cjs 'public/**/*.{tsx,ts}' 'packages/grafana-ui/**/*.{tsx,ts}' && yarn i18n:pseudo",
-    "i18n:pseudo": "node ./public/locales/pseudo.js",
     "i18n:stats": "node ./scripts/cli/reportI18nStats.mjs",
     "betterer": "betterer",
     "betterer:json": "ts-node --transpile-only --project ./scripts/cli/tsconfig.json ./scripts/cli/bettererResultsToJson.ts",

public/app/core/internationalization/constants.ts

@@ -1,4 +1,5 @@
 import { ResourceKey } from 'i18next';
+import { uniq } from 'lodash';
 
 export const ENGLISH_US = 'en-US';
 export const FRENCH_FRANCE = 'fr-FR';
@@ -10,7 +11,9 @@ export const PSEUDO_LOCALE = 'pseudo-LOCALE';
 
 export const DEFAULT_LANGUAGE = ENGLISH_US;
 
-interface LanguageDefinitions {
+export type LocaleFileLoader = () => Promise<ResourceKey>;
+
+export interface LanguageDefinition<Namespace extends string = string> {
   /** IETF language tag for the language e.g. en-US */
   code: string;
 
@@ -18,53 +21,90 @@ interface LanguageDefinitions {
   name: string;
 
   /** Function to load translations */
-  loader: () => Promise<ResourceKey>;
+  loader: Record<Namespace, LocaleFileLoader>;
 }
 
-export const LANGUAGES: LanguageDefinitions[] = [
+export const LANGUAGES: LanguageDefinition[] = [
   {
     code: ENGLISH_US,
     name: 'English',
-    loader: () => import('../../../locales/en-US/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/en-US/grafana.json'),
+    },
   },
 
   {
     code: FRENCH_FRANCE,
     name: 'Français',
-    loader: () => import('../../../locales/fr-FR/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/fr-FR/grafana.json'),
+    },
   },
 
   {
     code: SPANISH_SPAIN,
     name: 'Español',
-    loader: () => import('../../../locales/es-ES/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/es-ES/grafana.json'),
+    },
   },
 
   {
     code: GERMAN_GERMANY,
     name: 'Deutsch',
-    loader: () => import('../../../locales/de-DE/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/de-DE/grafana.json'),
+    },
   },
 
   {
     code: CHINESE_SIMPLIFIED,
     name: '中文（简体）',
-    loader: () => import('../../../locales/zh-Hans/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/zh-Hans/grafana.json'),
+    },
   },
 
   {
     code: BRAZILIAN_PORTUGUESE,
     name: 'Português Brasileiro',
-    loader: () => import('../../../locales/pt-BR/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/pt-BR/grafana.json'),
+    },
   },
-];
+] satisfies Array<LanguageDefinition<'grafana'>>;
 
 if (process.env.NODE_ENV === 'development') {
   LANGUAGES.push({
     code: PSEUDO_LOCALE,
     name: 'Pseudo-locale',
-    loader: () => import('../../../locales/pseudo-LOCALE/grafana.json'),
+    loader: {
+      grafana: () => import('../../../locales/pseudo-LOCALE/grafana.json'),
+    },
   });
 }
 
+// Optionally load enterprise locale extensions, if they are present.
+// It is important that this happens before NAMESPACES is defined so it has the correct value
+//
+// require.context doesn't work in jest, so we don't even attempt to load enterprise translations...
+if (process.env.NODE_ENV !== 'test') {
+  const extensionRequireContext = require.context('../../', true, /app\/extensions\/locales\/localeExtensions/);
+  if (extensionRequireContext.keys().includes('app/extensions/locales/localeExtensions')) {
+    const { LOCALE_EXTENSIONS, ENTERPRISE_I18N_NAMESPACE } = extensionRequireContext(
+      'app/extensions/locales/localeExtensions'
+    );
+
+    for (const language of LANGUAGES) {
+      const localeLoader = LOCALE_EXTENSIONS[language.code];
+
+      if (localeLoader) {
+        language.loader[ENTERPRISE_I18N_NAMESPACE] = localeLoader;
+      }
+    }
+  }
+}
+
 export const VALID_LANGUAGES = LANGUAGES.map((v) => v.code);
+
+export const NAMESPACES = uniq(LANGUAGES.flatMap((v) => Object.keys(v.loader)));

public/app/core/internationalization/index.tsx

@@ -3,12 +3,12 @@ import LanguageDetector, { DetectorOptions } from 'i18next-browser-languagedetec
 import React from 'react';
 import { Trans as I18NextTrans, initReactI18next } from 'react-i18next'; // eslint-disable-line no-restricted-imports
 
-import { DEFAULT_LANGUAGE, VALID_LANGUAGES } from './constants';
+import { DEFAULT_LANGUAGE, NAMESPACES, VALID_LANGUAGES } from './constants';
 import { loadTranslations } from './loadTranslations';
 
 let tFunc: TFunction<string[], undefined> | undefined;
 
-export function initializeI18n(language: string): Promise<{ language: string | undefined }> {
+export async function initializeI18n(language: string): Promise<{ language: string | undefined }> {
   // This is a placeholder so we can put a 'comment' in the message json files.
   // Starts with an underscore so it's sorted to the top of the file. Even though it is in a comment the following line is still extracted
   // t('_comment', 'The code is the source of truth for English phrases. They should be updated in the components directly, and additional plurals specified in this file.');
@@ -24,7 +24,10 @@ export function initializeI18n(language: string): Promise<{ language: string | u
     // Required to ensure that `resolvedLanguage` is set property when an invalid language is passed (such as through 'detect')
     supportedLngs: VALID_LANGUAGES,
     fallbackLng: DEFAULT_LANGUAGE,
+
+    ns: NAMESPACES,
   };
+
   let i18nInstance = i18n;
   if (language === 'detect') {
     i18nInstance = i18nInstance.use(LanguageDetector);
@@ -39,13 +42,13 @@ export function initializeI18n(language: string): Promise<{ language: string | u
     .use(initReactI18next) // passes i18n down to react-i18next
     .init(options);
 
-  tFunc = i18n.t;
+  await loadPromise;
 
-  return loadPromise.then(() => {
-    return {
-      language: i18nInstance.resolvedLanguage,
-    };
-  });
+  tFunc = i18n.getFixedT(null, NAMESPACES);
+
+  return {
+    language: i18nInstance.resolvedLanguage,
+  };
 }
 
 export function changeLanguage(locale: string) {
@@ -54,7 +57,7 @@ export function changeLanguage(locale: string) {
 }
 
 export const Trans: typeof I18NextTrans = (props) => {
-  return <I18NextTrans shouldUnescape {...props} />;
+  return <I18NextTrans shouldUnescape ns={NAMESPACES} {...props} />;
 };
 
 // Wrap t() to provide default namespaces and enforce a consistent API

public/app/core/internationalization/loadTranslations.ts

@@ -12,10 +12,17 @@ export const loadTranslations: BackendModule = {
     if (!localeDef) {
       localeDef = LANGUAGES.find((v) => getLanguagePartFromCode(v.code) === getLanguagePartFromCode(language));
     }
+
     if (!localeDef) {
-      return callback(new Error('No message loader available for ' + language), null);
+      return callback(new Error(`No message loader available for ${language}`), null);
     }
-    const messages = await localeDef.loader();
+
+    const namespaceLoader = localeDef.loader[namespace];
+    if (!namespaceLoader) {
+      return callback(new Error(`No message loader available for ${language} with namespace ${namespace}`), null);
+    }
+
+    const messages = await namespaceLoader();
     callback(null, messages);
   },
 };

public/locales/i18next-parser-enterprise.config.cjs

@@ -0,0 +1,8 @@
+const baseConfig = require('./i18next-parser.config.cjs');
+
+module.exports = {
+  ...baseConfig,
+  defaultNamespace: 'grafana-enterprise',
+  input: ['../../public/app/extensions/**/*.{tsx,ts}'],
+  output: './public/app/extensions/locales/$LOCALE/$NAMESPACE.json',
+};

public/locales/i18next-parser.config.cjs

@@ -1,6 +1,6 @@
 module.exports = {
-  // Base config
-  locales: ['en-US'], // Only en-US is updated - Crowdin will PR with other languages
+  // Base config - same for both OSS and Enterprise
+  locales: ['en-US'], // Only en-US  is updated - Crowdin will PR with other languages
   sort: true,
   createOldCatalogs: false,
   failOnWarnings: true,
@@ -9,6 +9,10 @@ module.exports = {
 
   // OSS-specific config
   defaultNamespace: 'grafana',
-  input: ['../../public/**/*.{tsx,ts}', '!../../public/app/extensions/**/*', '../../packages/grafana-ui/**/*.{tsx,ts}'],
+  input: [
+    '../../public/**/*.{tsx,ts}',
+    '!../../public/app/extensions/**/*', // Don't extract from Enterprise
+    '../../packages/grafana-ui/**/*.{tsx,ts}',
+  ],
   output: './public/locales/$LOCALE/$NAMESPACE.json',
 };

public/locales/pseudo.js

@@ -1,24 +0,0 @@
-const fs = require('fs/promises');
-const pseudoizer = require('pseudoizer');
-const prettier = require('prettier');
-
-function pseudoizeJsonReplacer(key, value) {
-  if (typeof value === 'string') {
-    // Split string on brace-enclosed segments. Odd indices will be {{variables}}
-    const phraseParts = value.split(/(\{\{[^}]+}\})/g);
-    const translatedParts = phraseParts.map((str, index) => index % 2 ? str : pseudoizer.pseudoize(str))
-    return translatedParts.join("")
-  }
-
-  return value;
-}
-
-fs.readFile('./public/locales/en-US/grafana.json').then(async (enJson) => {
-  const enMessages = JSON.parse(enJson);
-  // Add newline to make prettier happy
-  const pseudoJson = await prettier.format(JSON.stringify(enMessages, pseudoizeJsonReplacer, 2), {
-    parser: 'json',
-  });
-
-  return fs.writeFile('./public/locales/pseudo-LOCALE/grafana.json', pseudoJson);
-});

public/locales/pseudo.mjs

@@ -0,0 +1,63 @@
+// @ts-check
+import { readFile, writeFile } from 'fs/promises';
+import { format } from 'prettier';
+import { pseudoize } from 'pseudoizer';
+import { hideBin } from 'yargs/helpers';
+import yargs from 'yargs/yargs';
+
+const argv = await yargs(hideBin(process.argv))
+  .option('mode', {
+    demandOption: true,
+    describe: 'Path to a template to use for each issue. See source bettererIssueTemplate.md for an example',
+    type: 'string',
+    choices: ['oss', 'enterprise', 'both'],
+  })
+  .version(false).argv;
+
+const extractOSS = ['oss', 'both'].includes(argv.mode);
+const extractEnterprise = ['enterprise', 'both'].includes(argv.mode);
+
+/**
+ * @param {string} key
+ * @param {unknown} value
+ */
+function pseudoizeJsonReplacer(key, value) {
+  if (typeof value === 'string') {
+    // Split string on brace-enclosed segments. Odd indices will be {{variables}}
+    const phraseParts = value.split(/(\{\{[^}]+}\})/g);
+    const translatedParts = phraseParts.map((str, index) => (index % 2 ? str : pseudoize(str)));
+    return translatedParts.join('');
+  }
+
+  return value;
+}
+/**
+ * @param {string} inputPath
+ * @param {string} outputPath
+ */
+async function pseudoizeJson(inputPath, outputPath) {
+  const baseJson = await readFile(inputPath, 'utf-8');
+  const enMessages = JSON.parse(baseJson);
+  const pseudoJson = JSON.stringify(enMessages, pseudoizeJsonReplacer, 2);
+  const prettyPseudoJson = await format(pseudoJson, {
+    parser: 'json',
+  });
+
+  await writeFile(outputPath, prettyPseudoJson);
+  console.log('Wrote', outputPath);
+}
+
+//
+// OSS translations
+if (extractOSS) {
+  await pseudoizeJson('./public/locales/en-US/grafana.json', './public/locales/pseudo-LOCALE/grafana.json');
+}
+
+//
+// Enterprise translations
+if (extractEnterprise) {
+  await pseudoizeJson(
+    './public/app/extensions/locales/en-US/grafana-enterprise.json',
+    './public/app/extensions/locales/pseudo-LOCALE/grafana-enterprise.json'
+  );
+}

scripts/drone/steps/lib.star

@@ -641,7 +641,7 @@ def lint_frontend_step():
 
 def verify_i18n_step():
     extract_error_message = "\nExtraction failed. Make sure that you have no dynamic translation phrases, such as 't(\\`preferences.theme.\\$${themeID}\\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file."
-    uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again."
+    uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'make i18n-extract', commit the changes and push again."
     return {
         "name": "verify-i18n",
         "image": images["node"],
@@ -651,7 +651,7 @@ def verify_i18n_step():
         "failure": "ignore",
         "commands": [
             "apk add --update git",
-            "yarn run i18n:extract || (echo \"{}\" && false)".format(extract_error_message),
+            "make i18n-extract || (echo \"{}\" && false)".format(extract_error_message),
             # Verify that translation extraction has been committed
             '''
             file_diff=$(git diff --dirstat public/locales)

yarn.lock

@@ -18365,8 +18365,8 @@ __metadata:
   linkType: hard
 
 "glob-stream@npm:^8.0.0":
-  version: 8.0.0
-  resolution: "glob-stream@npm:8.0.0"
+  version: 8.0.2
+  resolution: "glob-stream@npm:8.0.2"
   dependencies:
     "@gulpjs/to-absolute-glob": "npm:^4.0.0"
     anymatch: "npm:^3.1.3"
@@ -18376,7 +18376,7 @@ __metadata:
     is-negated-glob: "npm:^1.0.0"
     normalize-path: "npm:^3.0.0"
     streamx: "npm:^2.12.5"
-  checksum: 10/b1d18b6fd49086ff02e031f03e3debac747047d304b349a6dced3b7944c665344ef63496363f483acc7c6afcd6ebfb11af1652824f2c370d83c0f3905d5c67e0
+  checksum: 10/cda46c02b6313d4a5cd0a3e67c7a2bd477d5f708904dc761c0d6364611f188a303051ec4e0cd405597522c7f7ffbba530f147754b4bf5af9f18e970c024734d8
   languageName: node
   linkType: hard