FIX: i18n integrity specs

FIX: check all .yml files in the project for integrity
FIX: ensure localized yamls are compatible with english

config/locales/server.ar.yml

@@ -2120,8 +2120,6 @@ ar:
   performance_report:
     initial_post_raw: 'هذا الموضوع يحتوي على معلومات الاداء اليومي للموقع '
     initial_topic_title: التبليغ عن اداء الموقع
-  topic_invite:
-    user_exists: "آسف، ذلك المستخدم قد تمت دعوته من قبل. تستطيع فقط أن تدعوا عضواً لموضوعِ مرة واحدة."
   activemodel:
     errors:
       <<: *errors

config/locales/server.da.yml

@@ -1197,8 +1197,6 @@ da:
   performance_report:
     initial_post_raw: Dette emne inkluderer daglige præstationsrapporter for dit site.
     initial_topic_title: Website præstationsrapport
-  topic_invite:
-    user_exists: "Beklager, men brugeren er allerede inviteret. Du kan kun invitere en bruger til et emne én gang."
   tags:
     title: "Tags"
     staff_tag_disallowed: "Tagget \"%{tag}\" kan kun tildeles af personalet."

config/locales/server.de.yml

@@ -2900,8 +2900,6 @@ de:
   performance_report:
     initial_post_raw: Dieser Beitrag enthält tägliche Leistungsberichte deiner Site.
     initial_topic_title: Berichte zur Websitegeschwindigkeit
-  topic_invite:
-    user_exists: "Entschuldige, dieser Benutzer ist bereits eingeladen worden. Du kannst einen Benutzer nur einmal zu einem Thema einladen."
   tags:
     title: "Schlagwörter"
     staff_tag_disallowed: "Das Schlagwort \"%{tag}\" darf nur vom Team verwendet werden."

config/locales/server.es.yml

@@ -2695,8 +2695,6 @@ es:
   performance_report:
     initial_post_raw: Este tema contiene informes diarios sobre el rendimiento de tu sito.
     initial_topic_title: Informe sobre el rendimiento del sitio
-  topic_invite:
-    user_exists: "Lo sentimos, ese usuario ya ha sido invitado. Solo se puede invitar una vez a un usuario a un tema."
   tags:
     title: "Etiquetas"
     staff_tag_disallowed: "La etiqueta \"%{tag}\" solo puede ser insertada por moderadores."

config/locales/server.fi.yml

@@ -2652,8 +2652,6 @@ fi:
   performance_report:
     initial_post_raw: Tämä ketju sisältää päivittäisiä suorituskykyrapotteja sivustoltasi
     initial_topic_title: Sivuston suorituskykyraportit
-  topic_invite:
-    user_exists: "Pahoittelut, tämä käyttäjä on jo kutsuttu. Voit kutsua toisen käyttäjän ketjuun vain yhden kerran."
   tags:
     title: "Tunnisteet"
     staff_tag_disallowed: "Tunnisteen \"%{tag}\" voi lisätä vain henkilökunta"

config/locales/server.fr.yml

@@ -2665,8 +2665,6 @@ fr:
   performance_report:
     initial_post_raw: Ce sujet comprend des rapports de performance journaliers concernant votre site.
     initial_topic_title: Rapports de performances du site
-  topic_invite:
-    user_exists: "Désolé, cet utilisateur a déjà été invité. Vous ne pouvez inviter un utilisateur qu'une seule fois par sujet."
   tags:
     title: "Tags"
     staff_tag_disallowed: "Le tag \"%{tag}\" ne peut être mis que par un responsable."

config/locales/server.he.yml

@@ -2904,8 +2904,6 @@ he:
   performance_report:
     initial_post_raw: 'נושא זה כולל דוחות פעילות יומיים עבור האתר שלך. '
     initial_topic_title: דוחות פעילות לאתר
-  topic_invite:
-    user_exists: "מצטערים, המשתמשים כבר הוזמנו. ניתן להזמין משתמשים לנושא רק פעם אחת."
   tags:
     title: "תגיות"
     staff_tag_disallowed: "התג \"%{tag}\" ניתן רק על ידי הצוות."

config/locales/server.nl.yml

@@ -1570,8 +1570,6 @@ nl:
   performance_report:
     initial_post_raw: Deze topic bevat dagelijkse performancerapporten van je site
     initial_topic_title: Performancerapportages van de website
-  topic_invite:
-    user_exists: "Sorry, die gebruiker is al uitgenodigd. Je kan een gebruiker maar een keer voor een topic uitnodigen."
   safe_mode:
     no_customizations: "Alle website-aanpassingen uitschakelen"
     only_official: "Niet-officiële plug-ins uitschakelen"

config/locales/server.pt.yml

@@ -2747,8 +2747,6 @@ pt:
   performance_report:
     initial_post_raw: Este tópico inclui relatórios diários de desempenho para o seu sítio.
     initial_topic_title: Relatórios de desempenho do sítio
-  topic_invite:
-    user_exists: "Pedimos desculpa, esse utilizador já foi convidado. Pode convidar um utilizador para um tópico apenas uma vez."
   tags:
     title: "Etiquetas"
     staff_tag_disallowed: "A etiqueta \"%{tag}\" pode ser aplicada pela equipa de apoio apenas."

config/locales/server.pt_BR.yml

@@ -1882,8 +1882,6 @@ pt_BR:
   performance_report:
     initial_post_raw: Este tópico inclui relatórios de performance diários de seu site.
     initial_topic_title: Relatórios de performance do Site
-  topic_invite:
-    user_exists: "Desculpe, este usuário já foi convidado. Você pode convidar um usuário para um tópico apenas uma única vez."
   tags:
     title: "Marcações"
     staff_tag_disallowed: "A marcação \"%{tag}\" pode ser aplicada somente pelo pessoal de apoio."

config/locales/server.ro.yml

@@ -2803,8 +2803,6 @@ ro:
   performance_report:
     initial_post_raw: Acest subiect include rapoarte zilnice de performanță referitoare la site-ul tău.
     initial_topic_title: Rapoarte de performanță website
-  topic_invite:
-    user_exists: "Ne pare rău, acest utilizator a fost deja invitat. Nu poți să inviți un utilizator la un subiect decât o singură dată."
   tags:
     title: "Etichete"
     staff_tag_disallowed: "Eticheta \"%{tag}\" poate fi pusă doar de un membru al echipei."

config/locales/server.ru.yml

@@ -1833,8 +1833,6 @@ ru:
   performance_report:
     initial_post_raw: Эта тема содержит ежедневные отчеты активности форума.
     initial_topic_title: Отчеты активности форума
-  topic_invite:
-    user_exists: "К сожалению, этот пользователь уже был приглашён. Вы можете пригласить пользователя в тему только один раз."
   tags:
     title: "Теги"
     staff_tag_disallowed: "Тег \"%{tag}\" может быть применён только персоналом."

config/locales/server.sv.yml

@@ -2306,8 +2306,6 @@ sv:
   performance_report:
     initial_post_raw: Det här ämnet innehåller dagliga prestandarapporter för din webbplats.
     initial_topic_title: Prestandarapporter för webbplatsen
-  topic_invite:
-    user_exists: "Tyvärr, den användaren har redan bjudits in. Du kan endast bjuda in en användare till ett ämne en gång."
   tags:
     title: "Taggar"
     staff_tag_disallowed: "Taggen \"%{tag}\" kan endast användas av personalen."

config/locales/server.tr_TR.yml

@@ -2048,8 +2048,6 @@ tr_TR:
   performance_report:
     initial_post_raw: Bu konu siteniz hakkında günlük performans raporlarını içerir.
     initial_topic_title: Site performansı raporları
-  topic_invite:
-    user_exists: "Üzgünüz, bu kullanıcı zaten davet edildi. Konuya yalnızca bir kullanıcı davet edebilirsiniz."
   tags:
     title: "Etiketler"
     staff_tag_disallowed: " \"%{tag}\" etiketi yalnızca görevliler tarafından eklenebilir gözüküyor."

config/locales/server.vi.yml

@@ -1913,8 +1913,6 @@ vi:
   performance_report:
     initial_post_raw: Chủ đề này bao gồm các báo cáo hiệu suất hàng ngày của website.
     initial_topic_title: Báo cáo hiệu suất website
-  topic_invite:
-    user_exists: "Xin lỗi, thành viên này đã được mời. Bạn chỉ có thể mời một người dùng đến một chủ đề một lần."
   tags:
     title: "Thẻ"
   activemodel:

config/locales/server.zh_CN.yml

@@ -2741,8 +2741,6 @@ zh_CN:
   performance_report:
     initial_post_raw: 这个主题将用来展示网站每日性能报告。
     initial_topic_title: 网站性能报告
-  topic_invite:
-    user_exists: "抱歉，用户已经被邀请了。你可能只想邀请用户参与主题一次。"
   tags:
     title: "标签"
     staff_tag_disallowed: "“%{tag}”只可由管理人员使用。"

config/locales/server.zh_TW.yml

@@ -2797,8 +2797,6 @@ zh_TW:
   performance_report:
     initial_post_raw: 這個主題將用來展示網站每日性能報告。
     initial_topic_title: 網站效能報表
-  topic_invite:
-    user_exists: "抱歉，用戶已經被邀請了。你可能只想邀請用戶參與主題一次。"
   tags:
     title: "標籤"
     staff_tag_disallowed: "“%{tag}”只可由管理人員使用。"

config/locales/transliterate.vi.yml

@@ -143,6 +143,3 @@ vi:
         Ỷ: "Y"
         Ỹ: "Y"
         Đ: "D"
-        ê: "e"
-        ù: "u"
-        à: "a"

lib/i18n/duplicate_key_finder.rb

@@ -0,0 +1,17 @@
+require_relative "locale_file_walker"
+
+class DuplicateKeyFinder < LocaleFileWalker
+
+  def find_duplicates(path)
+    @keys_with_count = Hash.new { 0 }
+    handle_document(Psych.parse_file(path))
+    @keys_with_count.select { |key, count| count > 1 }.keys
+  end
+
+  protected
+
+    def handle_scalar(node, depth, parents)
+      super
+      @keys_with_count[parents.join('.')] += 1
+    end
+end

lib/i18n/locale_file_walker.rb

@@ -1,28 +1,26 @@
-require 'psych'
-require 'set'
-
 class LocaleFileWalker
   protected
 
+  def handle_stream(stream)
+    stream.children.each { |document| handle_document(document) }
+  end
+
   def handle_document(document)
-    # we want to ignore the language (first key), so let's start at -1
+    # we want to ignore the locale (first key), so let's start at -1
     handle_nodes(document.root.children, -1, [])
   end
 
   def handle_nodes(nodes, depth, parents)
-    if nodes
-      consecutive_scalars = 0
-      nodes.each do |node|
-        consecutive_scalars = handle_node(node, depth, parents, consecutive_scalars)
-      end
+    return unless nodes
+    consecutive_scalars = 0
+    nodes.each do |node|
+      consecutive_scalars = handle_node(node, depth, parents, consecutive_scalars)
     end
   end
 
   def handle_node(node, depth, parents, consecutive_scalars)
-    node_is_scalar = node.is_a?(Psych::Nodes::Scalar)
-
-    if node_is_scalar
-      handle_scalar(node, depth, parents) if valid_scalar?(depth, consecutive_scalars)
+    if node_is_scalar = node.is_a?(Psych::Nodes::Scalar)
+      valid_scalar?(depth, consecutive_scalars) ? handle_scalar(node, depth, parents) : handle_value(node.value, parents)
     elsif node.is_a?(Psych::Nodes::Alias)
       handle_alias(node, depth, parents)
     elsif node.is_a?(Psych::Nodes::Mapping)
@@ -37,6 +35,9 @@ class LocaleFileWalker
     depth >= 0 && consecutive_scalars.even?
   end
 
+  def handle_value(value, parents)
+  end
+
   def handle_scalar(node, depth, parents)
     parents[depth] = node.value
   end

plugins/discourse-nginx-performance-report/config/locales/server.zn_CN.yml

@@ -1,3 +1,3 @@
-zh_CN:
+zn_CN:
   site_settings:
     daily_performance_report: "每日分析 NGINX 日志并且发布详情主题到管理人员才能看到的主题"

spec/integrity/i18n_spec.rb

@@ -1,127 +1,122 @@
-require 'rails_helper'
-require 'locale_file_walker'
+require "rails_helper"
+require "i18n/duplicate_key_finder"
+
+def extract_locale(path)
+  path[/\.([^.]{2,})\.yml$/, 1]
+end
+
+PLURALIZATION_KEYS ||= ['zero', 'one', 'two', 'few', 'many', 'other']
+
+def find_pluralizations(hash, parent_key = '', pluralizations = Hash.new)
+  hash.each do |key, value|
+    if Hash === value
+      current_key = parent_key.blank? ? key : "#{parent_key}.#{key}"
+      find_pluralizations(value, current_key, pluralizations)
+    elsif PLURALIZATION_KEYS.include? key
+      pluralizations[parent_key] = hash
+    end
+  end
+
+  pluralizations
+end
+
+def is_yaml_compatible?(english, translated)
+  english.each do |k, v|
+    if translated.has_key?(k)
+      if Hash === v
+        if Hash === translated[k]
+          return false unless is_yaml_compatible?(v, translated[k])
+        end
+      else
+        return false unless v.class == translated[k].class
+      end
+    end
+  end
+
+  true
+end
+
 
 describe "i18n integrity checks" do
 
-  it 'should have an i18n key for all trust levels' do
+  it 'has an i18n key for each Trust Levels' do
     TrustLevel.all.each do |ts|
       expect(ts.name).not_to match(/translation missing/)
     end
   end
 
-  it "needs an i18n key (description) for each Site Setting" do
+  it "has an i18n key for each Site Setting" do
     SiteSetting.all_settings.each do |s|
-      next if s[:setting] =~ /^test/
+      next if s[:setting][/^test_/]
       expect(s[:description]).not_to match(/translation missing/)
     end
   end
 
-  it "has an i18n key for each badge description" do
+  it "has an i18n key for each Badge description" do
     Badge.where(system: true).each do |b|
       expect(b.long_description).to be_present
       expect(b.description).to be_present
     end
   end
 
-  it "has valid YAML for client" do
-    Dir["#{Rails.root}/config/locales/client.*.yml"].each do |f|
-      locale = /.*\.([^.]{2,})\.yml$/.match(f)[1]
-      client = YAML.load_file("#{Rails.root}/config/locales/client.#{locale}.yml")
-      expect(client.count).to eq(1)
-      expect(client[locale]).not_to eq(nil)
-      expect(client[locale]["js"]).not_to eq(nil)
-      expect(client[locale]["admin_js"]).not_to eq(nil)
+  Dir["#{Rails.root}/config/locales/client.*.yml"].each do |path|
+    it "has valid client YAML for '#{path}'" do
+      yaml = YAML.load_file(path)
+      locale = extract_locale(path)
+
+      expect(yaml.keys).to eq([locale])
+
+      expect(yaml[locale]["js"]).to be
+      expect(yaml[locale]["admin_js"]).to be
+      # expect(yaml[locale]["wizard_js"]).to be
     end
   end
 
-  it "has valid YAML for server" do
-    Dir["#{Rails.root}/config/locales/server.*.yml"].each do |f|
-      locale = /.*\.([^.]{2,})\.yml$/.match(f)[1]
-      server = YAML.load_file("#{Rails.root}/config/locales/server.#{locale}.yml")
-      expect(server.count).to eq(1)
-      expect(server[locale]).not_to eq(nil)
-    end
-  end
+  Dir["#{Rails.root}/**/locale*/*.en.yml"].each do |english_path|
+    english_yaml = YAML.load_file(english_path)["en"]
 
-  it "does not overwrite another language" do
-    all = Dir["#{Rails.root}/config/locales/client.*.yml"] + Dir["#{Rails.root}/config/locales/server.*.yml"]
-    all.each do |f|
-      locale = /.*\.([^.]{2,})\.yml$/.match(f)[1] + ':'
-      IO.foreach(f) do |line|
-        line.strip!
-        next if line.start_with? "#"
-        next if line.start_with? "---"
-        next if line.blank?
-        expect(line).to eq locale
-        break
-      end
-    end
-  end
-
-  describe 'English locale file' do
-    locale_files = ['config/locales', 'plugins/**/locales']
-                     .product(['server.en.yml', 'client.en.yml'])
-                     .collect { |dir, filename| Dir["#{Rails.root}/#{dir}/#{filename}"] }
-                     .flatten
-                     .map { |path| Pathname.new(path).relative_path_from(Rails.root) }
-
-    class DuplicateKeyFinder < LocaleFileWalker
-      def find_duplicates(filename)
-        @keys_with_count = {}
-
-        document = Psych.parse_file(filename)
-        handle_document(document)
-
-        @keys_with_count.delete_if { |key, count| count <= 1 }.keys
+    context(english_path) do
+      it "has no duplicate keys" do
+        english_duplicates = DuplicateKeyFinder.new.find_duplicates(english_path)
+        expect(english_duplicates).to be_empty
       end
 
-      protected
+      find_pluralizations(english_yaml).each do |key, hash|
+        next if key["messages.restrict_dependent_destroy"]
 
-      def handle_scalar(node, depth, parents)
-        super(node, depth, parents)
-
-        key = parents.join('.')
-        @keys_with_count[key] = @keys_with_count.fetch(key, 0) + 1
-      end
-    end
-
-    module Pluralizations
-      def self.load(path)
-        whitelist = Regexp.union([/messages.restrict_dependent_destroy/])
-
-        yaml = YAML.load_file("#{Rails.root}/#{path}")
-        pluralizations = find_pluralizations(yaml['en'])
-        pluralizations.reject! { |key| key.match(whitelist) }
-        pluralizations
-      end
-
-      def self.find_pluralizations(hash, parent_key = '', pluralizations = Hash.new)
-        hash.each do |key, value|
-          if value.is_a? Hash
-            current_key = parent_key.blank? ? key : "#{parent_key}.#{key}"
-            find_pluralizations(value, current_key, pluralizations)
-          elsif key == 'one' || key == 'other'
-            pluralizations[parent_key] = hash
-          end
+        it "has valid pluralizations for '#{key}'" do
+          expect(hash.keys).to contain_exactly("one", "other")
         end
-
-        pluralizations
       end
     end
 
-    locale_files.each do |path|
-      context path do
-        it 'has no duplicate keys' do
-          duplicates = DuplicateKeyFinder.new.find_duplicates("#{Rails.root}/#{path}")
+    Dir[english_path.sub(".en.yml", ".*.yml")].each do |path|
+      next if path[".en.yml"]
+
+      context(path) do
+        locale = extract_locale(path)
+        yaml = YAML.load_file(path)
+
+        it "has no duplicate keys" do
+          duplicates = DuplicateKeyFinder.new.find_duplicates(path)
           expect(duplicates).to be_empty
         end
 
-        Pluralizations.load(path).each do |key, values|
-          it "key '#{key}' has valid pluralizations" do
-            expect(values.keys).to contain_exactly('one', 'other')
-          end
+        it "does not overwrite another locale" do
+          expect(yaml.keys).to eq([locale])
         end
+
+        unless path["transliterate"]
+
+          it "is compatible with english" do
+            expect(is_yaml_compatible?(english_yaml, yaml)).to eq(true)
+          end
+
+        end
+
       end
+
     end
   end
+
 end