Fix up translation issues

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/LanguageUtils.java

@@ -19,6 +19,8 @@ import org.hl7.fhir.r5.model.DataType;
 import org.hl7.fhir.r5.model.ElementDefinition;
 import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingAdditionalComponent;
 import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionConstraintComponent;
+import org.hl7.fhir.r5.model.Extension;
+import org.hl7.fhir.r5.model.MarkdownType;
 import org.hl7.fhir.r5.model.Resource;
 import org.hl7.fhir.r5.model.StringType;
 import org.hl7.fhir.r5.model.StructureDefinition;
@@ -56,8 +58,9 @@ public class LanguageUtils {
 
   private static final String ORPHAN_TRANSLATIONS_NAME = "translations.orphans";
 
-  private static final String SUPPLEMENT_NAME = "translations.supplement";
+  private static final String SUPPLEMENT_SOURCE_RESOURCE = "translations.supplemented";
-
+  private static final String SUPPLEMENT_SOURCE_TRANSLATIONS = "translations.source-list";
+  
   IWorkerContext context;
   private List<String> crlist;
   
@@ -332,18 +335,63 @@ public class LanguageUtils {
     return dstLang == null || srcLang == null ? false : dstLang.startsWith(srcLang) || "*".equals(srcLang);
   }
 
-  public static void fillSupplement(CodeSystem cs, List<TranslationUnit> list) {
+  public static void fillSupplement(CodeSystem csSrc, CodeSystem csDst, List<TranslationUnit> list) {
-    cs.setUserData(SUPPLEMENT_NAME, "true");
+    csDst.setUserData(SUPPLEMENT_SOURCE_RESOURCE, csSrc);
+    csDst.setUserData(SUPPLEMENT_SOURCE_TRANSLATIONS, list);
     for (TranslationUnit tu : list) {
-      ConceptDefinitionComponent cd = CodeSystemUtilities.getCode(cs, tu.getId());
+      String code = tu.getId();
-      if (cd != null && cd.hasDisplay() && cd.getDisplay().equals(tu.getSrcText())) {
+      String subCode = null;
-        cd.addDesignation().setLanguage(tu.getLanguage()).setValue(tu.getTgtText());
+      if (code.contains("@")) {
-      } else {
+        subCode = code.substring(code.indexOf("@")+1);
-        addOrphanTranslation(cs, tu);
+        code = code.substring(0, code.indexOf("@"));
       }
+      ConceptDefinitionComponent cdSrc = CodeSystemUtilities.getCode(csSrc, tu.getId());
+      if (cdSrc == null) {
+        addOrphanTranslation(csSrc, tu);
+      } else {
+        ConceptDefinitionComponent cdDst = CodeSystemUtilities.getCode(csDst, cdSrc.getCode());
+        if (cdDst == null) {
+          cdDst = csDst.addConcept().setCode(cdSrc.getCode());
+        }
+        String tt = tu.getTgtText();
+        if (tt.startsWith("!!")) {
+          tt = tt.substring(3);
+        }
+        if (subCode == null) {
+          cdDst.setDisplay(tt);
+        } else if ("definition".equals(subCode)) {
+          cdDst.setDefinition(tt);
+        } else {
+          boolean found = false;
+          for (ConceptDefinitionDesignationComponent d : cdSrc.getDesignation()) {
+            if (d.hasUse() && subCode.equals(d.getUse().getCode())) {
+              found = true;
+              cdDst.addDesignation().setUse(d.getUse()).setLanguage(tu.getLanguage()).setValue(tt); //.setUserData(SUPPLEMENT_SOURCE, tu);       
+              break;
+            }
+          }
+          if (!found) {
+            for (Extension e : cdSrc.getExtension()) {
+              if (subCode.equals(tail(e.getUrl()))) {
+                found = true;
+                cdDst.addExtension().setUrl(e.getUrl()).setValue(
+                    e.getValue().fhirType().equals("markdown") ? new MarkdownType(tt) : new StringType(tt)); //.setUserData(SUPPLEMENT_SOURCE, tu);          
+                break;
+              }
+            }
+          }
+          if (!found) {
+            addOrphanTranslation(csSrc, tu);            
+          }
+        }
+      }      
     }    
   }
 
+  private static Object tail(String url) {
+    return url.contains("/") ? url.substring(url.lastIndexOf("/")+1) : url;
+  }
+
   private static void addOrphanTranslation(CodeSystem cs, TranslationUnit tu) {
     List<TranslationUnit> list = (List<TranslationUnit>) cs.getUserData(ORPHAN_TRANSLATIONS_NAME);
     if (list == null) {
@@ -376,7 +424,7 @@ public class LanguageUtils {
   }
 
   public static boolean handlesAsResource(Resource resource) {
-    return (resource instanceof CodeSystem && resource.hasUserData(SUPPLEMENT_NAME)) || (resource instanceof StructureDefinition);
+    return (resource instanceof CodeSystem && resource.hasUserData(SUPPLEMENT_SOURCE_RESOURCE)) || (resource instanceof StructureDefinition);
   }
 
   public static boolean handlesAsElement(Element element) {
@@ -388,10 +436,23 @@ public class LanguageUtils {
     if (res instanceof StructureDefinition) {
       StructureDefinition sd = (StructureDefinition) res;
       generateTranslations(list, sd, lang);
+      if (res.hasUserData(ORPHAN_TRANSLATIONS_NAME)) {
+        List<TranslationUnit> orphans = (List<TranslationUnit>) res.getUserData(ORPHAN_TRANSLATIONS_NAME);
+        for (TranslationUnit t : orphans) {
+          list.add(new TranslationUnit(lang, "!!"+t.getId(), t.getContext1(), t.getSrcText(), t.getTgtText()));
+        }
+      }
     } else {
-      CodeSystem cs = (CodeSystem) res;
+      CodeSystem cs = (CodeSystem) res.getUserData(SUPPLEMENT_SOURCE_RESOURCE);
+      List<TranslationUnit> inputs = res.hasUserData(SUPPLEMENT_SOURCE_TRANSLATIONS) ? (List<TranslationUnit>) res.getUserData(SUPPLEMENT_SOURCE_TRANSLATIONS) : new ArrayList<>();
       for (ConceptDefinitionComponent cd : cs.getConcept()) {
-        generateTranslations(list, cd, lang);
+        generateTranslations(list, cd, lang, inputs);
+      }
+      if (cs.hasUserData(ORPHAN_TRANSLATIONS_NAME)) {
+        List<TranslationUnit> orphans = (List<TranslationUnit>) cs.getUserData(ORPHAN_TRANSLATIONS_NAME);
+        for (TranslationUnit t : orphans) {
+          list.add(new TranslationUnit(lang, "!!"+t.getId(), t.getContext1(), t.getSrcText(), t.getTgtText()));
+        }
       }
     }
     return list;
@@ -434,26 +495,41 @@ public class LanguageUtils {
     
   }
 
-  private static void generateTranslations(List<TranslationUnit> list, ConceptDefinitionComponent cd, String lang) {
+  private static void generateTranslations(List<TranslationUnit> list, ConceptDefinitionComponent cd, String lang, List<TranslationUnit> inputs) {
-    String code = cd.getCode();
+    // we generate translation units for the display, the definition, and any designations and extensions that we find
-    String display = cd.getDisplay();
+    // the id of the designation is the use.code (there will be a use) and for the extension, the tail of the extension URL 
-    String target = null;
+    // todo: do we need to worry about name clashes? why would we, and more importantly, how would we solve that?
-    for (ConceptDefinitionDesignationComponent d : cd.getDesignation()) {
+
-      if (target == null && !d.hasUse() && d.hasLanguage() && lang.equals(d.getLanguage())) {
+    addTranslationUnit(list, cd.getCode(), cd.getDisplay(), lang, inputs);
-        target = d.getValue();
+    if (cd.hasDefinition()) {
-      }
+      addTranslationUnit(list, cd.getCode()+"@definition", cd.getDefinition(), lang, inputs);        
     }
     for (ConceptDefinitionDesignationComponent d : cd.getDesignation()) {
-      if (target == null && d.hasLanguage() && lang.equals(d.getLanguage())) {
+      addTranslationUnit(list, cd.getCode()+"@"+d.getUse().getCode(), d.getValue(), lang, inputs);              
-        target = d.getValue();
-      }
     }
-    list.add(new TranslationUnit(lang, code, getDefinition(cd), display, target));
+    for (Extension e : cd.getExtension()) {
-    for (ConceptDefinitionComponent cd1 : cd.getConcept()) {
+      addTranslationUnit(list, cd.getCode()+"@"+tail(e.getUrl()), e.getValue().primitiveValue(), lang, inputs);                    
-      generateTranslations(list, cd1, lang);
     }
   }
 
+  private static void addTranslationUnit(List<TranslationUnit> list, String id, String srcText, String lang, List<TranslationUnit> inputs) {
+    TranslationUnit existing = null;
+    for (TranslationUnit t : inputs) {
+      if (id.equals(t.getId())) {
+        existing = t;
+        break;
+      }
+    }
+    // not sure what to do with context?
+    if (existing == null) {
+      list.add(new TranslationUnit(lang, id, null, srcText, null));
+    } else if (srcText.equals(existing.getSrcText())) {
+      list.add(new TranslationUnit(lang, id, null, srcText, existing.getTgtText()));
+    } else {
+      list.add(new TranslationUnit(lang, id, null, srcText, "!!"+existing.getTgtText()).setOriginal(existing.getSrcText()));
+    }    
+  }
+  
   private static String getDefinition(ConceptDefinitionComponent cd) {
     ConceptPropertyComponent v = CodeSystemUtilities.getProperty(cd, "translation-context");
     if (v != null && v.hasValue()) {

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/JsonLangFileProducer.java

@@ -130,6 +130,9 @@ public class JsonLangFileProducer extends LanguageFileProducer {
       if (tu.getContext1() != null) { 
         entry.add("context", tu.getContext1());
       }
+      if (tu.getOriginal() != null) {
+        entry.add("original", tu.getOriginal());
+      }
       entry.add("source", tu.getSrcText());
       entry.add("target", tu.getTgtText());
     }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/LanguageFileProducer.java

@@ -65,6 +65,7 @@ public abstract class LanguageFileProducer {
   
   public static class TranslationUnit extends TextUnit {
     private String language;
+    private String original; // for when the source text has been modified since being translated
 
     public TranslationUnit(String language, String id, String context, String srcText, String tgtText) {
       super(id, context, srcText, tgtText);
@@ -86,6 +87,16 @@ public abstract class LanguageFileProducer {
     public void setTgtText(String tgtText) {
       this.tgtText = tgtText;
     }
+
+    public String getOriginal() {
+      return original;
+    }
+
+    public TranslationUnit setOriginal(String original) {
+      this.original = original;
+      return this;
+    }
+    
   }
 
   public class Translations {

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/PoGetTextProducer.java

@@ -167,6 +167,9 @@ public class PoGetTextProducer extends LanguageFileProducer {
       if (tu.getContext1() != null) {
         ln(po, "#. "+tu.getContext1());
       }
+      if (tu.getOriginal() != null) {
+        ln(po, "#| "+tu.getOriginal());
+      }
       ln(po, "msgid \""+stripEoln(tu.getSrcText())+"\"");
       ln(po, "msgstr \""+(tu.getTgtText() == null ? "" : stripEoln(tu.getTgtText()))+"\"");
       ln(po, "");

pom.xml

@@ -21,7 +21,7 @@
         <commons_compress_version>1.26.0</commons_compress_version>
         <guava_version>32.0.1-jre</guava_version>
         <hapi_fhir_version>6.4.1</hapi_fhir_version>
-        <validator_test_case_version>1.5.7</validator_test_case_version>
+        <validator_test_case_version>1.5.8-SNAPSHOT</validator_test_case_version>
         <jackson_version>2.17.0</jackson_version>
         <junit_jupiter_version>5.9.2</junit_jupiter_version>
         <junit_platform_launcher_version>1.8.2</junit_platform_launcher_version>