Merge pull request #1332 from hapifhir/gg-202206-spdx-android

Gg 202206 spdx android

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java

@@ -57,6 +57,19 @@ public class VSACImporter extends OIDBasedValueSetImporter {
             errs.put(oid, "Expansion: " +e.getMessage());
             System.out.println(e.getMessage());
           }
+          if (vs.hasTitle()) {
+            if (vs.getTitle().equals(vs.getDescription())) {
+              vs.setTitle(vs.getName());              
+            } else {
+              System.out.println(oid);
+              System.out.println("  name: "+vs.getName());
+              System.out.println("  title: "+vs.getTitle());
+              System.out.println("  desc: "+vs.getDescription());
+            }
+          } else {
+            vs.setTitle(vs.getName());
+          }
+          vs.setName(makeValidName(vs.getName()));
           new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
         }
         i++;
@@ -75,4 +88,32 @@ public class VSACImporter extends OIDBasedValueSetImporter {
     new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo);
     System.out.println("Done. " + i + " ValueSets");
   }
+
+  private String makeValidName(String name) {
+    StringBuilder b = new StringBuilder();
+    boolean upper = true;
+    for (char ch : name.toCharArray()) {
+      if (ch == ' ') {
+        upper = true;
+      } else if (Character.isAlphabetic(ch)) {
+        if (upper) {
+          b.append(Character.toUpperCase(ch));
+        } else {
+          b.append(ch);
+        }
+        upper = false;
+      } else if (Character.isDigit(ch)) {
+        if (b.length() == 0) {
+          b.append('N');
+        }
+        b.append(ch);
+      } else if (ch == '_' && b.length() != 0) {
+        b.append(ch);
+      } else {
+        upper = true;
+      }
+    }
+    System.out.println(b.toString()+" from "+name);
+    return b.toString();
+  }
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/TextFile.java

@@ -62,13 +62,18 @@ public class TextFile {
 		List<String> result = new ArrayList<String>();
 		
 		File file = new CSFile(path);
-		BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file),"UTF-8"));
+		FileInputStream fs = new FileInputStream(file);
+		try {
+		  BufferedReader reader = new BufferedReader(new InputStreamReader(fs,"UTF-8"));
 
-		while( reader.ready() )
+		  while( reader.ready() )
-			result.add(reader.readLine());
+		    result.add(reader.readLine());
 
-		reader.close();
+		  reader.close();
-		return result;
+		  return result;
+		} finally {
+		  fs.close();
+		}
 	}
 	
 	public static void writeAllLines(String path, List<String> lines) throws IOException
@@ -141,12 +146,22 @@ public class TextFile {
   }
 
   public static String fileToString(File f) throws FileNotFoundException, IOException {
-    return streamToString(new FileInputStream(f));
+    FileInputStream fs = new FileInputStream(f);
+    try {
+      return streamToString(fs);
+    } finally {
+      fs.close();
+    }
   }
   
   public static String fileToString(String src) throws FileNotFoundException, IOException  {
-    return streamToString(new FileInputStream(new CSFile(src)));
+    FileInputStream fs = new FileInputStream(new CSFile(src));
-	}
+    try {
+      return streamToString(fs);
+    } finally {
+      fs.close();
+    }
+  }
 
   public static String streamToString(InputStream input) throws IOException  {
     InputStreamReader sr = new InputStreamReader(input, "UTF-8");    
@@ -205,7 +220,12 @@ public class TextFile {
   }
 
   public static byte[] fileToBytes(String srcFile) throws FileNotFoundException, IOException {
-    return streamToBytes(new FileInputStream(new CSFile(srcFile)));
+    FileInputStream fs = new FileInputStream(new CSFile(srcFile));
+    try {
+      return streamToBytes(fs);
+    } finally {
+      fs.close();
+    }
   }
 
   /**
@@ -218,11 +238,21 @@ public class TextFile {
    * @throws IOException
    */
   public static byte[] fileToBytesNCS(String srcFile) throws FileNotFoundException, IOException {
-    return streamToBytes(new FileInputStream(new File(srcFile)));
+    FileInputStream fs = new FileInputStream(new File(srcFile));
+    try {
+      return streamToBytes(fs);
+    } finally {
+      fs.close();
+    }
   }
 
   public static byte[] fileToBytes(File file) throws FileNotFoundException, IOException {
-    return streamToBytes(new FileInputStream(file));
+    FileInputStream fs = new FileInputStream(file);
+    try {
+      return streamToBytes(fs);
+    } finally {
+      fs.close();
+    }
   }
 
   public static String bytesToString(byte[] bs) throws IOException {

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java

@@ -2033,4 +2033,30 @@ public class Utilities {
     return i == 0 ? "" : s.substring(0, i+1);
   }
 
+public static void renameDirectory(String source, String dest) throws FHIRException, IOException {
+	File src = new File(source);
+	File dst = new File(dest);
+	if (!src.renameTo(dst)) {
+	  int i = 0;
+	  do {
+	    try {
+		  Thread.sleep(20);
+        } catch (Exception e) {
+          // nothing
+        }
+		System.gc();
+		i++;
+	  } while (!src.renameTo(dst) && i < 10);
+	  if (src.exists()) {
+		copyDirectory(source, dest, null);
+		try {
+		  src.delete();	
+		} catch (Exception e) {
+		  // nothing
+		}
+	  }
+	}
+	
+}
+
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java

@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.UUID;
 
 import org.apache.commons.io.FileUtils;
 import org.hl7.fhir.exceptions.FHIRException;
@@ -421,7 +422,9 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
   public NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException {
     checkValidVersionString(version, id);
     
-    NpmPackage npm = NpmPackage.fromPackage(packageTgzInputStream, sourceDesc, true);
+    String uuid = UUID.randomUUID().toString().toLowerCase();
+    String tempDir = Utilities.path(cacheFolder, uuid);
+    NpmPackage npm =  NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory);
 
     if (progress) {
       log("");
@@ -450,40 +453,18 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
           } catch (Throwable t) {
             log("Unable to clear directory: "+packRoot+": "+t.getMessage()+" - this may cause problems later");
           }
-
+          Utilities.renameDirectory(tempDir, packRoot);          
-          int i = 0;
-          int c = 0;
-          int size = 0;
-          for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
-            String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());
-            if (!(new File(dir).exists()))
-              Utilities.createDirectory(dir);
-            for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
-              String fn = Utilities.path(dir, Utilities.cleanFileName(fe.getKey()));
-              byte[] cnt = fe.getValue();
-              TextFile.bytesToFile(cnt, fn);
-              size = size + cnt.length;
-              i++;
-              if (progress && i % 50 == 0) {
-                c++;
-                logn(".");
-                if (c == 120) {
-                  log("");
-                  logn("  ");
-                  c = 2;
-                }
-              }
-            }
-          }
-
 
           IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
           ini.setTimeStampFormat("yyyyMMddhhmmss");
           ini.setTimestampProperty("packages", id + "#" + v, Timestamp.from(Instant.now()), null);
-          ini.setIntegerProperty("package-sizes", id + "#" + v, size, null);
+          ini.setIntegerProperty("package-sizes", id + "#" + v, npm.getSize(), null);
           ini.save();
           if (progress)
             log(" done.");
+        } else {
+          Utilities.clearDirectory(tempDir);
+          new File(tempDir).delete();
         }
         if (!id.equals(npm.getNpm().asString("name")) || !v.equals(npm.getNpm().asString("version"))) {
           if (!id.equals(npm.getNpm().asString("name"))) {

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java

@@ -36,6 +36,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -295,6 +296,7 @@ public class NpmPackage {
   private boolean changedByLoader; // internal qa only!
   private Map<String, Object> userData;
   private boolean minimalMemory;
+  private int size;
 
   /**
    * Constructor
@@ -444,6 +446,63 @@ public class NpmPackage {
     return res;
   }
   
+  public static NpmPackage extractFromTgz(InputStream tgz, String desc, String tempDir, boolean minimal) throws IOException {
+    String dest =  Utilities.path(tempDir, "package");
+    Utilities.createDirectory(dest);
+
+    int size = 0;
+    
+    GzipCompressorInputStream gzipIn;
+    try {
+      gzipIn = new GzipCompressorInputStream(tgz);
+    } catch (Exception e) {
+      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
+    }
+    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
+      TarArchiveEntry entry;
+
+      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
+        String n = entry.getName();
+        if (n.startsWith("package/")) {
+          n = n.substring(8);
+        }
+        if (n.contains("..")) {
+          throw new RuntimeException("Entry with an illegal name: " + n);
+        }
+        if (entry.isDirectory()) {
+          if (!Utilities.noString(n)) {
+            String dir = n.substring(0, n.length()-1);
+            Utilities.createDirectory(Utilities.path(dest, dir));
+          }
+        } else {
+          int count;
+          byte data[] = new byte[BUFFER_SIZE];
+          String filename = Utilities.path(dest, n);
+          String folder = Utilities.getDirectoryForFile(filename);
+          Utilities.createDirectory(folder);
+          FileOutputStream fos = new FileOutputStream(filename);
+          try (BufferedOutputStream dst = new BufferedOutputStream(fos, BUFFER_SIZE)) {
+            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
+              dst.write(data, 0, count);
+              size = size + count;
+            }
+          }
+          fos.close();
+        }
+      }
+    } 
+    try {
+      NpmPackage npm = NpmPackage.fromFolderMinimal(tempDir);
+      npm.setSize(size);
+      if (!minimal) {
+        npm.checkIndexed(desc);
+      }
+      return npm;
+    } catch (Exception e) {
+      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
+    } 
+  }
+  
   public void readStream(InputStream tgz, String desc, boolean progress) throws IOException {
     GzipCompressorInputStream gzipIn;
     try {
@@ -1303,5 +1362,16 @@ public class NpmPackage {
     return Utilities.path(path, "package", d);
   }
 
+  public boolean isMinimalMemory() {
+    return minimalMemory;
+  }
+
+  public int getSize() {
+    return size;
+  }
+
+  public void setSize(int size) {
+    this.size = size;
+  }
   
 }

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/MinimalMemoryTests.java

@@ -0,0 +1,31 @@
+package org.hl7.fhir.utilities.npm;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+import java.io.File;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.io.IOUtils;
+import org.hl7.fhir.utilities.Utilities;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class MinimalMemoryTests {
+
+  @Test
+  public void testFetch() throws IOException {
+    File folder = new File(Utilities.path("[tmp]", ".fhir-mm"));
+    if (folder.exists()) {
+      Utilities.clearDirectory(folder.getAbsolutePath());
+    } else {
+      Utilities.createDirectory(folder.getAbsolutePath());
+    }
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(folder.getAbsolutePath());
+    pcm.setMinimalMemory(true);
+    NpmPackage npm = pcm.loadPackage("hl7.fhir.us.core", "5.0.0");
+    Assertions.assertTrue(npm.isMinimalMemory());
+  }
+
+}