From ef8034180634a49025577379fbbac8ea178c6a1e Mon Sep 17 00:00:00 2001
From: Gerhard Schlager <mail@gerhard-schlager.at>
Date: Fri, 27 Apr 2018 19:01:34 +0200
Subject: [PATCH] Update pull_translations script to work with latest Transifex
 changes

* supports resources created with Transifex's YML handler version 3
* uses translations-manager gem
* makes sure that the locales supported by translations-manager are not out of sync
* update the lang_map in tx client config before pulling translations
---
 script/pull_translations.rb | 349 ++++--------------------------------
 1 file changed, 34 insertions(+), 315 deletions(-)
 mode change 100644 => 100755 script/pull_translations.rb

diff --git a/script/pull_translations.rb b/script/pull_translations.rb
old mode 100644
new mode 100755
index 8b5faedb413..4370b229032
--- a/script/pull_translations.rb
+++ b/script/pull_translations.rb
@@ -1,327 +1,46 @@
-# This script pulls translation files from Transifex and ensures they are in the format we need.
-# You need the Transifex client installed.
-# http://docs.transifex.com/developer/client/setup
-#
-# Don't use this script to create pull requests. Do translations in Transifex. The Discourse
-# team will pull them in.
+#!/usr/bin/env ruby
 
-require 'open3'
-require 'psych'
-require 'set'
-require 'fileutils'
-require_relative '../lib/i18n/locale_file_walker'
+require 'bundler/inline'
+
+gemfile(true) do
+  gem 'translations-manager', git: 'https://github.com/discourse/translations-manager.git'
+end
+
+require 'translations_manager'
+
+def expand_path(path)
+  File.expand_path("../../#{path}", __FILE__)
+end
+
+def supported_locales
+  Dir.glob(expand_path('config/locales/client.*.yml'))
+    .map { |x| x.split('.')[-2] }
+    .select { |x| x != 'en' }
+    .sort
+end
 
 YML_DIRS = ['config/locales',
             'plugins/poll/config/locales',
             'plugins/discourse-details/config/locales',
             'plugins/discourse-narrative-bot/config/locales',
             'plugins/discourse-nginx-performance-report/config/locales',
-            'plugins/discourse-presence/config/locales']
+            'plugins/discourse-presence/config/locales'].map { |dir| expand_path(dir) }
 YML_FILE_PREFIXES = ['server', 'client']
+TX_CONFIG = expand_path('.tx/config')
 
-if `which tx`.strip.empty?
-  puts '', 'The Transifex client needs to be installed to use this script.'
-  puts 'Instructions are here: http://docs.transifex.com/client/setup/'
-  puts '', 'On Mac:', ''
-  puts '  sudo easy_install pip'
-  puts '  sudo pip install transifex-client', ''
+if TranslationsManager::SUPPORTED_LOCALES != supported_locales
+  STDERR.puts <<~MESSAGE
+
+    The supported locales are out of sync.
+    Please update the TranslationsManager::SUPPORTED_LOCALES in translations-manager.
+    https://github.com/discourse/translations-manager
+
+    The following locales are currently supported by Discourse:
+
+  MESSAGE
+
+  STDERR.puts locales.map { |l| "'#{l}'" }.join(",\n")
   exit 1
 end
 
-if ARGV.include?('force')
-  STDERR.puts 'Usage:   ruby pull_translations.rb [languages]'
-  STDERR.puts 'Example: ruby pull_translations.rb de it', ''
-  exit 1
-end
-
-def get_languages
-  if ARGV.empty?
-    Dir.glob(File.expand_path('../../config/locales/client.*.yml', __FILE__))
-      .map { |x| x.split('.')[-2] }
-  else
-    ARGV
-  end
-end
-
-def yml_path(dir, prefix, language)
-  path = "../../#{dir}/#{prefix}.#{language}.yml"
-  File.expand_path(path, __FILE__)
-end
-
-def yml_path_if_exists(dir, prefix, language)
-  path = yml_path(dir, prefix, language)
-  File.exists?(path) ? path : nil
-end
-
-languages = get_languages.select { |x| x != 'en' }.sort
-
-# ensure that all locale files exists. tx doesn't create missing locale files during pull
-YML_DIRS.each do |dir|
-  YML_FILE_PREFIXES.each do |prefix|
-    next unless yml_path_if_exists(dir, prefix, 'en')
-
-    languages.each do |language|
-      filename = yml_path(dir, prefix, language)
-      FileUtils.touch(filename) unless File.exists?(filename)
-    end
-  end
-end
-
-puts 'Pulling new translations...', ''
-command = "tx pull --mode=developer --language=#{languages.join(',')} --force"
-
-return_value = Open3.popen2e(command) do |_, stdout_err, wait_thr|
-  while (line = stdout_err.gets)
-    puts line
-  end
-  wait_thr.value
-end
-
-puts ''
-
-unless return_value.success?
-  STDERR.puts 'Something failed. Check the output above.', ''
-  exit return_value.exitstatus
-end
-
-YML_FILE_COMMENTS = <<END
-# encoding: utf-8
-#
-# Never edit this file. It will be overwritten when translations are pulled from Transifex.
-#
-# To work with us on translations, join this project:
-# https://www.transifex.com/projects/p/discourse-org/
-END
-
-# Add comments to the top of files and replace the language (first key in YAML file)
-def update_file_header(filename, language)
-  lines = File.readlines(filename)
-  lines.collect! { |line| line.gsub!(/^[a-z_]+:( {})?$/i, "#{language}:\\1") || line }
-
-  File.open(filename, 'w+') do |f|
-    f.puts(YML_FILE_COMMENTS, '') unless lines[0][0] == '#'
-    f.puts(lines)
-  end
-end
-
-class YamlAliasFinder < LocaleFileWalker
-  def initialize
-    @anchors = {}
-    @aliases = Hash.new { |hash, key| hash[key] = [] }
-  end
-
-  def parse_file(filename)
-    document = Psych.parse_file(filename)
-    handle_document(document)
-    { anchors: @anchors, aliases: @aliases }
-  end
-
-  private
-
-  def handle_alias(node, depth, parents)
-    @aliases[node.anchor] << parents.dup
-  end
-
-  def handle_mapping(node, depth, parents)
-    if node.anchor
-      @anchors[parents.dup] = node.anchor
-    end
-  end
-end
-
-class YamlAliasSynchronizer < LocaleFileWalker
-  def initialize(original_alias_data)
-    @anchors = original_alias_data[:anchors]
-    @aliases = original_alias_data[:aliases]
-    @used_anchors = Set.new
-
-    calculate_required_keys
-  end
-
-  def add_to(filename)
-    stream = Psych.parse_stream(File.read(filename))
-    stream.children.each { |document| handle_document(document) }
-
-    add_aliases
-    write_yaml(stream, filename)
-  end
-
-  private
-
-  def calculate_required_keys
-    @required_keys = {}
-
-    @aliases.each_value do |key_sets|
-      key_sets.each do |keys|
-        until keys.empty?
-          add_needed_node(keys)
-          keys = keys.dup
-          keys.pop
-        end
-      end
-    end
-
-    add_needed_node([]) unless @required_keys.empty?
-  end
-
-  def add_needed_node(keys)
-    @required_keys[keys] = { mapping: nil, scalar: nil, alias: nil }
-  end
-
-  def write_yaml(stream, filename)
-    yaml = stream.to_yaml(nil, line_width: -1)
-
-    File.open(filename, 'w') do |file|
-      file.write(yaml)
-    end
-  end
-
-  def handle_scalar(node, depth, parents)
-    super(node, depth, parents)
-
-    if @required_keys.has_key?(parents)
-      @required_keys[parents][:scalar] = node
-    end
-  end
-
-  def handle_alias(node, depth, parents)
-    if @required_keys.has_key?(parents)
-      @required_keys[parents][:alias] = node
-    end
-  end
-
-  def handle_mapping(node, depth, parents)
-    if @anchors.has_key?(parents)
-      node.anchor = @anchors[parents]
-      @used_anchors.add(node.anchor)
-    end
-
-    if @required_keys.has_key?(parents)
-      @required_keys[parents][:mapping] = node
-    end
-  end
-
-  def add_aliases
-    @used_anchors.each do |anchor|
-      @aliases[anchor].each do |keys|
-        parents = []
-        parent_node = @required_keys[[]]
-
-        keys.each_with_index do |key, index|
-          parents << key
-          current_node = @required_keys[parents]
-          is_last = index == keys.size - 1
-          add_node(current_node, parent_node, key, is_last ? anchor : nil)
-          parent_node = current_node
-        end
-      end
-    end
-  end
-
-  def add_node(node, parent_node, scalar_name, anchor)
-    parent_mapping = parent_node[:mapping]
-    parent_mapping.children ||= []
-
-    if node[:scalar].nil?
-      node[:scalar] = Psych::Nodes::Scalar.new(scalar_name)
-      parent_mapping.children << node[:scalar]
-    end
-
-    if anchor.nil?
-      if node[:mapping].nil?
-        node[:mapping] = Psych::Nodes::Mapping.new
-        parent_mapping.children << node[:mapping]
-      end
-    elsif node[:alias].nil?
-      parent_mapping.children << Psych::Nodes::Alias.new(anchor)
-    end
-  end
-end
-
-def get_english_alias_data(dir, prefix)
-  filename = yml_path_if_exists(dir, prefix, 'en')
-  filename ? YamlAliasFinder.new.parse_file(filename) : nil
-end
-
-def add_anchors_and_aliases(english_alias_data, filename)
-  if english_alias_data
-    YamlAliasSynchronizer.new(english_alias_data).add_to(filename)
-  end
-end
-
-def fix_invalid_yml_keys(filename)
-  lines = File.readlines(filename)
-
-  # fix YAML keys which are on the wrong line
-  lines.each { |line| line.gsub!(/^(.*(?:'|"))(\s*\S*:)$/, "\\1\n\\2") }
-
-  File.open(filename, 'w+') do |f|
-    f.puts(lines)
-  end
-end
-
-def fix_invalid_yml(filename)
-  lines = File.readlines(filename)
-
-  lines.each_with_index do |line, i|
-    if line =~ /^\s+#.*$/i
-      # remove comments
-      lines[i] = nil
-    elsif line.strip.empty?
-      # remove empty lines
-      lines[i] = nil
-    elsif line =~ /^\s+.*: (?:""|''|)$/i
-      # remove lines which contain empty string values
-      lines[i] = nil
-    elsif line =~ /^(\s)*.*: \|$/i
-      next_line = i + 1 < lines.size ? lines[i + 1] : nil
-
-      if next_line.nil? || line[/^\s*/].size + 2 != next_line[/^\s*/].size
-        # remove lines which contain the value | without a string in the next line
-        lines[i] = nil
-      end
-    end
-  end.compact!
-
-  loop do
-    lines.each_with_index do |line, i|
-      if line =~ /^\s+\w*:\s*$/i
-        next_line = i + 1 < lines.size ? lines[i + 1] : nil
-
-        if next_line.nil? || line[/^\s*/].size >= next_line[/^\s*/].size
-          # remove lines which have an empty value and are not followed
-          # by a key on the next level
-          lines[i] = nil
-        end
-      end
-    end
-
-    break if lines.compact!.nil?
-  end
-
-  File.open(filename, 'w+') do |f|
-    f.puts(lines)
-  end
-end
-
-YML_DIRS.each do |dir|
-  YML_FILE_PREFIXES.each do |prefix|
-    english_alias_data = get_english_alias_data(dir, prefix)
-
-    languages.each do |language|
-      filename = yml_path_if_exists(dir, prefix, language)
-
-      if filename
-        # The following methods were added to handle a bug in Transifex's yml. Should not be needed now.
-        # fix_invalid_yml_keys(filename)
-        # fix_invalid_yml(filename)
-
-        # TODO check if this is still needed with recent Transifex changes
-        #      Nov 14, 2017: yup, still needed
-        add_anchors_and_aliases(english_alias_data, filename)
-
-        update_file_header(filename, language)
-      end
-    end
-  end
-end
+TranslationsManager::TransifexUpdater.new(YML_DIRS, YML_FILE_PREFIXES, *ARGV).perform(tx_config_filename: TX_CONFIG)