From 73f4b9f507b576b1c924ad094d9eae644875b2cb Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Thu, 5 Dec 2019 05:09:48 +1100 Subject: [PATCH] fix bugs downloading packages --- .../org/hl7/fhir/r5/test/NpmPackageTests.java | 12 ++++ .../hl7/fhir/utilities/cache/NpmPackage.java | 41 ++++++----- .../utilities/cache/PackageCacheManager.java | 71 +++++++------------ 3 files changed, 61 insertions(+), 63 deletions(-) diff --git a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/NpmPackageTests.java b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/NpmPackageTests.java index 27b1b7cd9..39ccf9962 100644 --- a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/NpmPackageTests.java +++ b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/NpmPackageTests.java @@ -2,6 +2,8 @@ package org.hl7.fhir.r5.test; import static org.junit.Assert.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -54,6 +56,16 @@ public class NpmPackageTests { checkNpm(npm); } + @Test + public void testSave() throws IOException { + NpmPackage npm = NpmPackage.fromPackage(TestingUtilities.loadTestResourceStream("npm", "test.format.old.tgz")); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + npm.save(bs); + npm = NpmPackage.fromPackage(new ByteArrayInputStream(bs.toByteArray())); + checkNpm(npm); + } + + private void checkNpm(NpmPackage npm) throws IOException { Assert.assertEquals(1, npm.list("other").size()); Assert.assertEquals("help.png", npm.list("other").get(0)); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java index 305aacc56..c5f56f207 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java @@ -84,7 +84,7 @@ public class NpmPackage { return ver.matches("^[0-9]+\\.[0-9]+\\.[0-9]+$"); } - private class NpmPackageFolder { + public class NpmPackageFolder { private String name; private Map> types = new HashMap<>(); private Map content = new HashMap(); @@ -96,6 +96,10 @@ public class NpmPackage { this.name = name; } + public String getName() { + return name; + } + public void readIndex(JsonObject index) { this.index = index; for (JsonElement e : index.getAsJsonArray("files")) { @@ -127,6 +131,10 @@ public class NpmPackage { return res; } + public Map getContent() { + return content; + } + public byte[] fetchFile(String file) throws FileNotFoundException, IOException { if (folder != null) { File f = new File(Utilities.path(folder.getAbsolutePath(), file)); @@ -263,18 +271,17 @@ public class NpmPackage { fos.close(); loadFile(n, fos.toByteArray()); } - } - i++; - if (progress && i % 50 == 0) { - c++; - System.out.print("."); - if (c == 120) { - System.out.println(""); - System.out.print(" "); - c = 2; + if (progress && i % 50 == 0) { + c++; + System.out.print("."); + if (c == 120) { + System.out.println(""); + System.out.print(" "); + c = 2; + } } - } - } + } + } try { npm = JsonTrackingParser.parseJson(folders.get("package").fetchFile("package.json")); } catch (Exception e) { @@ -569,8 +576,8 @@ public class NpmPackage { } /** special case when playing around inside the package **/ - public Map getContent() { - return folders.get("package").content; + public Map getFolders() { + return folders; } // @@ -624,9 +631,9 @@ public class NpmPackage { for (String s : folder.content.keySet()) { byte[] b = folder.content.get(s); String name = n+"/"+s; - indexer.seeFile(n, b); + indexer.seeFile(s, b); if (!s.equals(".index.json") && !s.equals("package.json")) { - TarArchiveEntry entry = new TarArchiveEntry(s); + TarArchiveEntry entry = new TarArchiveEntry(name); entry.setSize(b.length); tar.putArchiveEntry(entry); tar.write(b); @@ -634,7 +641,7 @@ public class NpmPackage { } } byte[] cnt = indexer.build().getBytes(Charset.forName("UTF-8")); - TarArchiveEntry entry = new TarArchiveEntry(n+".index.json"); + TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json"); entry.setSize(cnt.length); tar.putArchiveEntry(entry); tar.write(cnt); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java index a92044459..a9c911f7e 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java @@ -47,6 +47,7 @@ import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder; import org.hl7.fhir.utilities.json.JSONUtil; import com.google.gson.GsonBuilder; @@ -181,29 +182,7 @@ public class PackageCacheManager { return cacheFolder; } - private void analysePackage(String dir) throws IOException { - NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder(); - indexer.start(); - int i = 0; - int c = 0; - File[] packages = new File(Utilities.path(dir, "package")).listFiles(); - for (File f : packages) { - if (!f.isDirectory()) { - indexer.seeFile(f.getName(), TextFile.fileToBytes(f)); - } - i++; - if (progress && i % 50 == 0) { - c++; - System.out.print("."); - if (c == 120) { - System.out.println(""); - System.out.print(" "); - c = 2; - } - } - } - TextFile.stringToFile(indexer.build(), Utilities.path(dir, "package", ".index.json")); - } + @@ -531,8 +510,10 @@ public class PackageCacheManager { recordMap(npm.canonical(), npm.name()); - if (progress ) - System.out.print("|"); + if (progress ) { + System.out.println(); + System.out.print(" Installing: "); + } if (npm.name() == null || id == null || !id.equals(npm.name())) { if (!id.equals("hl7.fhir.r5.core")) {// temporary work around throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name()); @@ -548,30 +529,28 @@ public class PackageCacheManager { int i = 0; int c = 0; int size = 0; - for (Entry e : npm.getContent().entrySet()) { - String fn = Utilities.path(packRoot, "package", e.getKey()); - String dir = Utilities.getDirectoryForFile(fn); + for (Entry e : npm.getFolders().entrySet()) { + String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());; if (!(new File(dir).exists())) Utilities.createDirectory(dir); - TextFile.bytesToFile(e.getValue(), fn); - size = size + e.getValue().length; - i++; - if (progress && i % 50 == 0) { - c++; - System.out.print("."); - if (c == 120) { - System.out.println(""); - System.out.print(" "); - c = 2; - } - } + for (Entry fe : e.getValue().getContent().entrySet()) { + String fn = Utilities.path(dir, fe.getKey()); + byte[] cnt = fe.getValue(); + TextFile.bytesToFile(cnt, fn); + size = size + cnt.length; + i++; + if (progress && i % 50 == 0) { + c++; + System.out.print("."); + if (c == 120) { + System.out.println(""); + System.out.print(" "); + c = 2; + } + } + } } - if (progress) { - System.out.println(""); - System.out.print(" Analysing"); - } - - analysePackage(packRoot); // do this whether there's an index.json or not. + IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); ini.setTimeStampFormat("yyyyMMddhhmmss");