mirror of
https://github.com/discourse/discourse.git
synced 2026-07-29 15:54:48 -05:00
Update pull_translations script to work with latest Transifex changes
* supports resources created with Transifex's YML handler version 3 * uses translations-manager gem * makes sure that the locales supported by translations-manager are not out of sync * update the lang_map in tx client config before pulling translations
This commit is contained in:
Gerhard Schlager
Regular → Executable
+34
-315
@@ -1,327 +1,46 @@
|
|||||||
# This script pulls translation files from Transifex and ensures they are in the format we need.
|
#!/usr/bin/env ruby
|
||||||
# You need the Transifex client installed.
|
|
||||||
# http://docs.transifex.com/developer/client/setup
|
|
||||||
#
|
|
||||||
# Don't use this script to create pull requests. Do translations in Transifex. The Discourse
|
|
||||||
# team will pull them in.
|
|
||||||
|
|
||||||
require 'open3'
|
require 'bundler/inline'
|
||||||
require 'psych'
|
|
||||||
require 'set'
|
gemfile(true) do
|
||||||
require 'fileutils'
|
gem 'translations-manager', git: 'https://github.com/discourse/translations-manager.git'
|
||||||
require_relative '../lib/i18n/locale_file_walker'
|
end
|
||||||
|
|
||||||
|
require 'translations_manager'
|
||||||
|
|
||||||
|
def expand_path(path)
|
||||||
|
File.expand_path("../../#{path}", __FILE__)
|
||||||
|
end
|
||||||
|
|
||||||
|
def supported_locales
|
||||||
|
Dir.glob(expand_path('config/locales/client.*.yml'))
|
||||||
|
.map { |x| x.split('.')[-2] }
|
||||||
|
.select { |x| x != 'en' }
|
||||||
|
.sort
|
||||||
|
end
|
||||||
|
|
||||||
YML_DIRS = ['config/locales',
|
YML_DIRS = ['config/locales',
|
||||||
'plugins/poll/config/locales',
|
'plugins/poll/config/locales',
|
||||||
'plugins/discourse-details/config/locales',
|
'plugins/discourse-details/config/locales',
|
||||||
'plugins/discourse-narrative-bot/config/locales',
|
'plugins/discourse-narrative-bot/config/locales',
|
||||||
'plugins/discourse-nginx-performance-report/config/locales',
|
'plugins/discourse-nginx-performance-report/config/locales',
|
||||||
'plugins/discourse-presence/config/locales']
|
'plugins/discourse-presence/config/locales'].map { |dir| expand_path(dir) }
|
||||||
YML_FILE_PREFIXES = ['server', 'client']
|
YML_FILE_PREFIXES = ['server', 'client']
|
||||||
|
TX_CONFIG = expand_path('.tx/config')
|
||||||
|
|
||||||
if `which tx`.strip.empty?
|
if TranslationsManager::SUPPORTED_LOCALES != supported_locales
|
||||||
puts '', 'The Transifex client needs to be installed to use this script.'
|
STDERR.puts <<~MESSAGE
|
||||||
puts 'Instructions are here: http://docs.transifex.com/client/setup/'
|
|
||||||
puts '', 'On Mac:', ''
|
The supported locales are out of sync.
|
||||||
puts ' sudo easy_install pip'
|
Please update the TranslationsManager::SUPPORTED_LOCALES in translations-manager.
|
||||||
puts ' sudo pip install transifex-client', ''
|
https://github.com/discourse/translations-manager
|
||||||
|
|
||||||
|
The following locales are currently supported by Discourse:
|
||||||
|
|
||||||
|
MESSAGE
|
||||||
|
|
||||||
|
STDERR.puts locales.map { |l| "'#{l}'" }.join(",\n")
|
||||||
exit 1
|
exit 1
|
||||||
end
|
end
|
||||||
|
|
||||||
if ARGV.include?('force')
|
TranslationsManager::TransifexUpdater.new(YML_DIRS, YML_FILE_PREFIXES, *ARGV).perform(tx_config_filename: TX_CONFIG)
|
||||||
STDERR.puts 'Usage: ruby pull_translations.rb [languages]'
|
|
||||||
STDERR.puts 'Example: ruby pull_translations.rb de it', ''
|
|
||||||
exit 1
|
|
||||||
end
|
|
||||||
|
|
||||||
def get_languages
|
|
||||||
if ARGV.empty?
|
|
||||||
Dir.glob(File.expand_path('../../config/locales/client.*.yml', __FILE__))
|
|
||||||
.map { |x| x.split('.')[-2] }
|
|
||||||
else
|
|
||||||
ARGV
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def yml_path(dir, prefix, language)
|
|
||||||
path = "../../#{dir}/#{prefix}.#{language}.yml"
|
|
||||||
File.expand_path(path, __FILE__)
|
|
||||||
end
|
|
||||||
|
|
||||||
def yml_path_if_exists(dir, prefix, language)
|
|
||||||
path = yml_path(dir, prefix, language)
|
|
||||||
File.exists?(path) ? path : nil
|
|
||||||
end
|
|
||||||
|
|
||||||
languages = get_languages.select { |x| x != 'en' }.sort
|
|
||||||
|
|
||||||
# ensure that all locale files exists. tx doesn't create missing locale files during pull
|
|
||||||
YML_DIRS.each do |dir|
|
|
||||||
YML_FILE_PREFIXES.each do |prefix|
|
|
||||||
next unless yml_path_if_exists(dir, prefix, 'en')
|
|
||||||
|
|
||||||
languages.each do |language|
|
|
||||||
filename = yml_path(dir, prefix, language)
|
|
||||||
FileUtils.touch(filename) unless File.exists?(filename)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
puts 'Pulling new translations...', ''
|
|
||||||
command = "tx pull --mode=developer --language=#{languages.join(',')} --force"
|
|
||||||
|
|
||||||
return_value = Open3.popen2e(command) do |_, stdout_err, wait_thr|
|
|
||||||
while (line = stdout_err.gets)
|
|
||||||
puts line
|
|
||||||
end
|
|
||||||
wait_thr.value
|
|
||||||
end
|
|
||||||
|
|
||||||
puts ''
|
|
||||||
|
|
||||||
unless return_value.success?
|
|
||||||
STDERR.puts 'Something failed. Check the output above.', ''
|
|
||||||
exit return_value.exitstatus
|
|
||||||
end
|
|
||||||
|
|
||||||
YML_FILE_COMMENTS = <<END
|
|
||||||
# encoding: utf-8
|
|
||||||
#
|
|
||||||
# Never edit this file. It will be overwritten when translations are pulled from Transifex.
|
|
||||||
#
|
|
||||||
# To work with us on translations, join this project:
|
|
||||||
# https://www.transifex.com/projects/p/discourse-org/
|
|
||||||
END
|
|
||||||
|
|
||||||
# Add comments to the top of files and replace the language (first key in YAML file)
|
|
||||||
def update_file_header(filename, language)
|
|
||||||
lines = File.readlines(filename)
|
|
||||||
lines.collect! { |line| line.gsub!(/^[a-z_]+:( {})?$/i, "#{language}:\\1") || line }
|
|
||||||
|
|
||||||
File.open(filename, 'w+') do |f|
|
|
||||||
f.puts(YML_FILE_COMMENTS, '') unless lines[0][0] == '#'
|
|
||||||
f.puts(lines)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
class YamlAliasFinder < LocaleFileWalker
|
|
||||||
def initialize
|
|
||||||
@anchors = {}
|
|
||||||
@aliases = Hash.new { |hash, key| hash[key] = [] }
|
|
||||||
end
|
|
||||||
|
|
||||||
def parse_file(filename)
|
|
||||||
document = Psych.parse_file(filename)
|
|
||||||
handle_document(document)
|
|
||||||
{ anchors: @anchors, aliases: @aliases }
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def handle_alias(node, depth, parents)
|
|
||||||
@aliases[node.anchor] << parents.dup
|
|
||||||
end
|
|
||||||
|
|
||||||
def handle_mapping(node, depth, parents)
|
|
||||||
if node.anchor
|
|
||||||
@anchors[parents.dup] = node.anchor
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
class YamlAliasSynchronizer < LocaleFileWalker
|
|
||||||
def initialize(original_alias_data)
|
|
||||||
@anchors = original_alias_data[:anchors]
|
|
||||||
@aliases = original_alias_data[:aliases]
|
|
||||||
@used_anchors = Set.new
|
|
||||||
|
|
||||||
calculate_required_keys
|
|
||||||
end
|
|
||||||
|
|
||||||
def add_to(filename)
|
|
||||||
stream = Psych.parse_stream(File.read(filename))
|
|
||||||
stream.children.each { |document| handle_document(document) }
|
|
||||||
|
|
||||||
add_aliases
|
|
||||||
write_yaml(stream, filename)
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def calculate_required_keys
|
|
||||||
@required_keys = {}
|
|
||||||
|
|
||||||
@aliases.each_value do |key_sets|
|
|
||||||
key_sets.each do |keys|
|
|
||||||
until keys.empty?
|
|
||||||
add_needed_node(keys)
|
|
||||||
keys = keys.dup
|
|
||||||
keys.pop
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
add_needed_node([]) unless @required_keys.empty?
|
|
||||||
end
|
|
||||||
|
|
||||||
def add_needed_node(keys)
|
|
||||||
@required_keys[keys] = { mapping: nil, scalar: nil, alias: nil }
|
|
||||||
end
|
|
||||||
|
|
||||||
def write_yaml(stream, filename)
|
|
||||||
yaml = stream.to_yaml(nil, line_width: -1)
|
|
||||||
|
|
||||||
File.open(filename, 'w') do |file|
|
|
||||||
file.write(yaml)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def handle_scalar(node, depth, parents)
|
|
||||||
super(node, depth, parents)
|
|
||||||
|
|
||||||
if @required_keys.has_key?(parents)
|
|
||||||
@required_keys[parents][:scalar] = node
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def handle_alias(node, depth, parents)
|
|
||||||
if @required_keys.has_key?(parents)
|
|
||||||
@required_keys[parents][:alias] = node
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def handle_mapping(node, depth, parents)
|
|
||||||
if @anchors.has_key?(parents)
|
|
||||||
node.anchor = @anchors[parents]
|
|
||||||
@used_anchors.add(node.anchor)
|
|
||||||
end
|
|
||||||
|
|
||||||
if @required_keys.has_key?(parents)
|
|
||||||
@required_keys[parents][:mapping] = node
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def add_aliases
|
|
||||||
@used_anchors.each do |anchor|
|
|
||||||
@aliases[anchor].each do |keys|
|
|
||||||
parents = []
|
|
||||||
parent_node = @required_keys[[]]
|
|
||||||
|
|
||||||
keys.each_with_index do |key, index|
|
|
||||||
parents << key
|
|
||||||
current_node = @required_keys[parents]
|
|
||||||
is_last = index == keys.size - 1
|
|
||||||
add_node(current_node, parent_node, key, is_last ? anchor : nil)
|
|
||||||
parent_node = current_node
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def add_node(node, parent_node, scalar_name, anchor)
|
|
||||||
parent_mapping = parent_node[:mapping]
|
|
||||||
parent_mapping.children ||= []
|
|
||||||
|
|
||||||
if node[:scalar].nil?
|
|
||||||
node[:scalar] = Psych::Nodes::Scalar.new(scalar_name)
|
|
||||||
parent_mapping.children << node[:scalar]
|
|
||||||
end
|
|
||||||
|
|
||||||
if anchor.nil?
|
|
||||||
if node[:mapping].nil?
|
|
||||||
node[:mapping] = Psych::Nodes::Mapping.new
|
|
||||||
parent_mapping.children << node[:mapping]
|
|
||||||
end
|
|
||||||
elsif node[:alias].nil?
|
|
||||||
parent_mapping.children << Psych::Nodes::Alias.new(anchor)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def get_english_alias_data(dir, prefix)
|
|
||||||
filename = yml_path_if_exists(dir, prefix, 'en')
|
|
||||||
filename ? YamlAliasFinder.new.parse_file(filename) : nil
|
|
||||||
end
|
|
||||||
|
|
||||||
def add_anchors_and_aliases(english_alias_data, filename)
|
|
||||||
if english_alias_data
|
|
||||||
YamlAliasSynchronizer.new(english_alias_data).add_to(filename)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def fix_invalid_yml_keys(filename)
|
|
||||||
lines = File.readlines(filename)
|
|
||||||
|
|
||||||
# fix YAML keys which are on the wrong line
|
|
||||||
lines.each { |line| line.gsub!(/^(.*(?:'|"))(\s*\S*:)$/, "\\1\n\\2") }
|
|
||||||
|
|
||||||
File.open(filename, 'w+') do |f|
|
|
||||||
f.puts(lines)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def fix_invalid_yml(filename)
|
|
||||||
lines = File.readlines(filename)
|
|
||||||
|
|
||||||
lines.each_with_index do |line, i|
|
|
||||||
if line =~ /^\s+#.*$/i
|
|
||||||
# remove comments
|
|
||||||
lines[i] = nil
|
|
||||||
elsif line.strip.empty?
|
|
||||||
# remove empty lines
|
|
||||||
lines[i] = nil
|
|
||||||
elsif line =~ /^\s+.*: (?:""|''|)$/i
|
|
||||||
# remove lines which contain empty string values
|
|
||||||
lines[i] = nil
|
|
||||||
elsif line =~ /^(\s)*.*: \|$/i
|
|
||||||
next_line = i + 1 < lines.size ? lines[i + 1] : nil
|
|
||||||
|
|
||||||
if next_line.nil? || line[/^\s*/].size + 2 != next_line[/^\s*/].size
|
|
||||||
# remove lines which contain the value | without a string in the next line
|
|
||||||
lines[i] = nil
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end.compact!
|
|
||||||
|
|
||||||
loop do
|
|
||||||
lines.each_with_index do |line, i|
|
|
||||||
if line =~ /^\s+\w*:\s*$/i
|
|
||||||
next_line = i + 1 < lines.size ? lines[i + 1] : nil
|
|
||||||
|
|
||||||
if next_line.nil? || line[/^\s*/].size >= next_line[/^\s*/].size
|
|
||||||
# remove lines which have an empty value and are not followed
|
|
||||||
# by a key on the next level
|
|
||||||
lines[i] = nil
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
break if lines.compact!.nil?
|
|
||||||
end
|
|
||||||
|
|
||||||
File.open(filename, 'w+') do |f|
|
|
||||||
f.puts(lines)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
YML_DIRS.each do |dir|
|
|
||||||
YML_FILE_PREFIXES.each do |prefix|
|
|
||||||
english_alias_data = get_english_alias_data(dir, prefix)
|
|
||||||
|
|
||||||
languages.each do |language|
|
|
||||||
filename = yml_path_if_exists(dir, prefix, language)
|
|
||||||
|
|
||||||
if filename
|
|
||||||
# The following methods were added to handle a bug in Transifex's yml. Should not be needed now.
|
|
||||||
# fix_invalid_yml_keys(filename)
|
|
||||||
# fix_invalid_yml(filename)
|
|
||||||
|
|
||||||
# TODO check if this is still needed with recent Transifex changes
|
|
||||||
# Nov 14, 2017: yup, still needed
|
|
||||||
add_anchors_and_aliases(english_alias_data, filename)
|
|
||||||
|
|
||||||
update_file_header(filename, language)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|||||||
Reference in New Issue
Block a user