IntenseWebs/sphinx
3
0 Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2026-10-13 00:42:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

Refactor `utils/babel_runner.py`

Browse Source
This commit is contained in:
Adam Turner committed 2023-03-15 14:35:03 +00:00
1 parent c2529e0a5e
commit 4d6aa53ae4
2 files changed
+177 -140
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

babel.cfg
-35
View File
@@ -1,35 +0,0 @@
# How to setup this file
# https://babel.pocoo.org/en/latest/installation.html
# this file description:
# https://babel.pocoo.org/en/latest/messages.html
# Extraction from Python source files
[python: **.py]
encoding = utf-8
# Extraction from Jinja2 template files
[jinja2: **/templates/latex/**.tex_t]
variable_start_string = <%=
variable_end_string = %>
block_start_string = <%
block_end_string = %>
# Extraction from Jinja2 template files
[jinja2: **/templates/latex/**.sty_t]
variable_start_string = <%=
variable_end_string = %>
block_start_string = <%
block_end_string = %>
# Extraction from Jinja2 HTML templates
[jinja2: **/themes/**.html]
encoding = utf-8
ignore_tags = script,style
include_attrs = alt title summary
# Extraction from Jinja2 XML templates
[jinja2: **/themes/**.xml]
# Extraction from JavaScript files
[javascript: **.js]
[javascript: **.js_t]
utils/babel_runner.py
+177 -105
View File
@@ -17,17 +17,137 @@ import json
import logging
import os
import sys
import tempfile
from babel.messages.frontend import compile_catalog, extract_messages, update_catalog
from babel.messages.pofile import read_po
from babel.messages.catalog import Catalog
from babel.messages.extract import (
DEFAULT_KEYWORDS,
extract,
extract_javascript,
extract_python,
)
from babel.messages.mofile import write_mo
from babel.messages.pofile import read_po, write_po
from babel.util import pathmatch
from jinja2.ext import babel_extract as extract_jinja2
import sphinx
ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", ".."))
TEX_DELIMITERS = {
'variable_start_string': '<%=',
'variable_end_string': '%>',
'block_start_string': '<%',
'block_end_string': '%>',
}
METHOD_MAP = [
# Extraction from Python source files
('**.py', extract_python),
# Extraction from Jinja2 template files
('**/templates/latex/**.tex_t', extract_jinja2),
('**/templates/latex/**.sty_t', extract_jinja2),
# Extraction from Jinja2 HTML templates
('**/themes/**.html', extract_jinja2),
# Extraction from Jinja2 XML templates
('**/themes/**.xml', extract_jinja2),
# Extraction from JavaScript files
('**.js', extract_javascript),
('**.js_t', extract_javascript),
]
OPTIONS_MAP = {
# Extraction from Python source files
'**.py': {
'encoding': 'utf-8',
},
# Extraction from Jinja2 template files
'**/templates/latex/**.tex_t': TEX_DELIMITERS.copy(),
'**/templates/latex/**.sty_t': TEX_DELIMITERS.copy(),
# Extraction from Jinja2 HTML templates
'**/themes/**.html': {
'encoding': 'utf-8',
'ignore_tags': 'script,style',
'include_attrs': 'alt title summary',
},
}
KEYWORDS = {**DEFAULT_KEYWORDS, '_': None, '__': None}
class compile_catalog_plusjs(compile_catalog):
def run_extract():
"""Message extraction function."""
log = _get_logger()
input_path = 'sphinx'
catalogue = Catalog(project='Sphinx', version=sphinx.__version__, charset='utf-8')
base = os.path.abspath(input_path)
for root, dirnames, filenames in os.walk(base):
relative_root = os.path.relpath(root, base) if root != base else ''
dirnames.sort()
for filename in sorted(filenames):
relative_name = os.path.join(relative_root, filename)
for pattern, method in METHOD_MAP:
if not pathmatch(pattern, relative_name):
continue
options = {}
for opt_pattern, opt_dict in OPTIONS_MAP.items():
if pathmatch(opt_pattern, relative_name):
options = opt_dict
with open(os.path.join(root, filename), 'rb') as fileobj:
for lineno, message, comments, context in extract(
method, fileobj, KEYWORDS, options=options,
):
filepath = os.path.join(input_path, relative_name)
catalogue.add(
message, None, [(filepath, lineno)],
auto_comments=comments, context=context,
)
break
output_file = os.path.join('sphinx', 'locale', 'sphinx.pot')
log.info('writing PO template file to %s', output_file)
with open(output_file, 'wb') as outfile:
write_po(outfile, catalogue)
def run_update():
"""Catalog merging command."""
log = _get_logger()
domain = 'sphinx'
locale_dir = os.path.join('sphinx', 'locale')
template_file = os.path.join(locale_dir, 'sphinx.pot')
with open(template_file, encoding='utf-8') as infile:
template = read_po(infile)
for locale in os.listdir(locale_dir):
filename = os.path.join(locale_dir, locale, 'LC_MESSAGES', f'{domain}.po')
if not os.path.exists(filename):
continue
log.info('updating catalog %s based on %s', filename, template_file)
with open(filename, encoding='utf-8') as infile:
catalog = read_po(infile, locale=locale, domain=domain)
catalog.update(template)
tmp_name = os.path.join(
os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename),
)
try:
with open(tmp_name, 'wb') as tmpfile:
write_po(tmpfile, catalog)
except Exception:
os.remove(tmp_name)
raise
os.replace(tmp_name, filename)
def run_compile():
"""
Catalog compilation command.
An extended command that writes all message strings that occur in
JavaScript files to a JavaScript file along with the .mo file.
@@ -35,72 +155,59 @@ class compile_catalog_plusjs(compile_catalog):
most of the run() code is duplicated here.
"""
def run(self):
if super().run():
print("Compiling failed.", file=sys.stderr)
raise SystemExit(2)
log = _get_logger()
for domain in self.domain:
self._run_domain_js(domain)
directory = os.path.join('sphinx', 'locale')
total_errors = 0
def _run_domain_js(self, domain):
po_files = []
js_files = []
for locale in os.listdir(directory):
po_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.po')
if not os.path.exists(po_file):
continue
if not self.input_file:
if self.locale:
po_files.append((self.locale,
os.path.join(self.directory, self.locale,
'LC_MESSAGES',
domain + '.po')))
js_files.append(os.path.join(self.directory, self.locale,
'LC_MESSAGES',
domain + '.js'))
else:
for locale in os.listdir(self.directory):
po_file = os.path.join(self.directory, locale,
'LC_MESSAGES',
domain + '.po')
if os.path.exists(po_file):
po_files.append((locale, po_file))
js_files.append(os.path.join(self.directory, locale,
'LC_MESSAGES',
domain + '.js'))
else:
po_files.append((self.locale, self.input_file))
if self.output_file:
js_files.append(self.output_file)
else:
js_files.append(os.path.join(self.directory, self.locale,
'LC_MESSAGES',
domain + '.js'))
with open(po_file, encoding='utf-8') as infile:
catalog = read_po(infile, locale)
for js_file, (locale, po_file) in zip(js_files, po_files):
with open(po_file, encoding='utf8') as infile:
catalog = read_po(infile, locale)
if catalog.fuzzy:
log.info('catalog %s is marked as fuzzy, skipping', po_file)
continue
if catalog.fuzzy and not self.use_fuzzy:
continue
for message, errors in catalog.check():
for error in errors:
total_errors += 1
log.error('error: %s:%d: %s', po_file, message.lineno, error)
self.log.info('writing JavaScript strings in catalog %s to %s',
po_file, js_file)
mo_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.mo')
log.info('compiling catalog %s to %s', po_file, mo_file)
with open(mo_file, 'wb') as outfile:
write_mo(outfile, catalog, use_fuzzy=False)
jscatalog = {}
for message in catalog:
if any(x[0].endswith(('.js', '.js_t', '.html'))
for x in message.locations):
msgid = message.id
if isinstance(msgid, (list, tuple)):
msgid = msgid[0]
jscatalog[msgid] = message.string
js_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.js')
log.info('writing JavaScript strings in catalog %s to %s', po_file, js_file)
js_catalogue = {}
for message in catalog:
if any(
x[0].endswith(('.js', '.js.jinja', '.js_t', '.html'))
for x in message.locations
):
msgid = message.id
if isinstance(msgid, (list, tuple)):
msgid = msgid[0]
js_catalogue[msgid] = message.string
obj = json.dumps({
'messages': jscatalog,
'plural_expr': catalog.plural_expr,
'locale': f'{catalog.locale!s}',
}, sort_keys=True, indent=4)
with open(js_file, 'w', encoding='utf8') as outfile:
outfile.write(f'Documentation.addTranslations({obj});')
obj = json.dumps({
'messages': js_catalogue,
'plural_expr': catalog.plural_expr,
'locale': str(catalog.locale),
}, sort_keys=True, indent=4)
with open(js_file, 'wb') as outfile:
# to ensure lines end with ``\n`` rather than ``\r\n``:
outfile.write(f'Documentation.addTranslations({obj});'.encode())
if total_errors > 0:
log.error('%d errors encountered.', total_errors)
print("Compiling failed.", file=sys.stderr)
raise SystemExit(2)
def _get_logger():
@@ -112,50 +219,6 @@ def _get_logger():
return log
def run_extract():
os.chdir(ROOT)
command = extract_messages()
command.log = _get_logger()
command.initialize_options()
command.keywords = "_ __ l_ lazy_gettext"
command.mapping_file = "babel.cfg"
command.output_file = os.path.join("sphinx", "locale", "sphinx.pot")
command.project = "Sphinx"
command.version = sphinx.__version__
command.input_paths = "sphinx"
command.finalize_options()
return command.run()
def run_update():
os.chdir(ROOT)
command = update_catalog()
command.log = _get_logger()
command.initialize_options()
command.domain = "sphinx"
command.input_file = os.path.join("sphinx", "locale", "sphinx.pot")
command.output_dir = os.path.join("sphinx", "locale")
command.finalize_options()
return command.run()
def run_compile():
os.chdir(ROOT)
command = compile_catalog_plusjs()
command.log = _get_logger()
command.initialize_options()
command.domain = "sphinx"
command.directory = os.path.join("sphinx", "locale")
command.finalize_options()
return command.run()
if __name__ == '__main__':
try:
action = sys.argv[1].lower()
@@ -163,9 +226,18 @@ if __name__ == '__main__':
print(__doc__, file=sys.stderr)
raise SystemExit(2)
os.chdir(ROOT)
if action == "extract":
raise SystemExit(run_extract())
if action == "update":
raise SystemExit(run_update())
if action == "compile":
raise SystemExit(run_compile())
if action == "all":
exit_code = run_extract()
if exit_code:
raise SystemExit(exit_code)
exit_code = run_update()
if exit_code:
raise SystemExit(exit_code)
raise SystemExit(run_compile())