From 0615f2a16364edfc9292f4718743e9efd1528326 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Thu, 22 Jun 2023 15:05:59 +1000 Subject: [PATCH] Minimal Memory model for NpmPackage --- .../npm/FilesystemPackageCacheManager.java | 2 +- .../hl7/fhir/utilities/npm/NpmPackage.java | 157 ++++++++++++------ .../utilities/tests/PackageCacheTests.java | 21 +++ 3 files changed, 132 insertions(+), 48 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java index bd1a194d2..2ce2a2edc 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java @@ -197,7 +197,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } private NpmPackage loadPackageInfo(String path) throws IOException { - NpmPackage pi = NpmPackage.fromFolder(path); + NpmPackage pi = minimalMemory ? NpmPackage.fromFolderMinimal(path) : NpmPackage.fromFolder(path); return pi; } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java index dd1aaa66d..a42960fe7 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java @@ -68,6 +68,7 @@ import org.hl7.fhir.utilities.SimpleHTTPClient; import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.json.JsonException; import org.hl7.fhir.utilities.json.model.JsonArray; import org.hl7.fhir.utilities.json.model.JsonElement; import org.hl7.fhir.utilities.json.model.JsonObject; @@ -162,36 +163,52 @@ public class NpmPackage { } public class NpmPackageFolder { - private String name; - private Map> types = new HashMap<>(); - private Map content = new HashMap<>(); - private JsonObject index; + private String folderName; + private Map> types; + private Map content; + private JsonObject cachedIndex; private File folder; - public NpmPackageFolder(String name) { + public NpmPackageFolder(String folderName) { super(); - this.name = name; + this.folderName = folderName; + if (!minimalMemory) { + types = new HashMap<>(); + content = new HashMap<>(); + } } - public Map> getTypes() { - return types; + private String fn(String name) throws IOException { + return Utilities.path(folder.getAbsolutePath(), name); } - public String getName() { - return name; + public Map> getTypes() throws JsonException, IOException { + if (minimalMemory) { + Map> typeMap = new HashMap<>(); + readIndex(JsonParser.parseObjectFromFile(fn(".index.json")), typeMap); + return typeMap; + } else { + return types; + } } - public boolean readIndex(JsonObject index) { + public String getFolderName() { + return folderName; + } + + public boolean readIndex(JsonObject index, Map> typeMap) { if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) { return false; } - this.index = index; + if (!minimalMemory) { + this.cachedIndex = index; + } for (JsonObject file : index.getJsonObjects("files")) { String type = file.asString("resourceType"); String name = file.asString("filename"); - if (!types.containsKey(type)) - types.put(type, new ArrayList<>()); - types.get(type).add(name); + if (!typeMap.containsKey(type)) + typeMap.put(type, new ArrayList<>()); + typeMap.get(type).add(name); } return true; } @@ -216,6 +233,8 @@ public class NpmPackage { } public Map getContent() { + assert !minimalMemory; + return content; } @@ -242,7 +261,7 @@ public class NpmPackage { } public String dump() { - return name + " ("+ (folder == null ? "null" : folder.toString())+") | "+Boolean.toString(index != null)+" | "+content.size()+" | "+types.size(); + return folderName + " ("+ (folder == null ? "null" : folder.toString())+")"+(minimalMemory ? "" : " | "+Boolean.toString(cachedIndex != null)+" | "+content.size()+" | "+types.size()); } public void removeFile(String n) throws IOException { @@ -254,13 +273,28 @@ public class NpmPackage { changedByLoader = true; } + public JsonObject index() throws IOException { + if (cachedIndex != null) { + return cachedIndex; + } else if (folder == null) { + return null; + } else { + File ij = new File(fn(".index.json")); + if (ij.exists()) { + return JsonParser.parseObject(ij); + } else { + return null; + } + } + } } private String path; private JsonObject npm; private Map folders = new HashMap<>(); private boolean changedByLoader; // internal qa only! - private Map userData = new HashMap<>(); + private Map userData; + private boolean minimalMemory; /** * Constructor @@ -279,6 +313,17 @@ public class NpmPackage { return res; } + /** + * Factory method that parses a package from an extracted folder + */ + public static NpmPackage fromFolderMinimal(String path) throws IOException { + NpmPackage res = new NpmPackage(); + res.minimalMemory = true; + res.loadFiles(path, new File(path)); + res.checkIndexed(path); + return res; + } + /** * Factory method that starts a new empty package using the given PackageGenerator to create the manifest */ @@ -297,6 +342,9 @@ public class NpmPackage { } public Map getUserData() { + if (userData == null) { + userData = new HashMap<>(); + } return userData; } @@ -312,17 +360,19 @@ public class NpmPackage { if (!d.equals("package")) { d = Utilities.path("package", d); } - NpmPackageFolder folder = this.new NpmPackageFolder(d); - folder.folder = f; - this.folders.put(d, folder); File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json")); if (ij.exists()) { - try { - if (!folder.readIndex(JsonParser.parseObject(ij))) { - indexFolder(folder.getName(), folder); + NpmPackageFolder folder = this.new NpmPackageFolder(d); + folder.folder = f; + this.folders.put(d, folder); + if (!minimalMemory) { + try { + if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) { + indexFolder(folder.getFolderName(), folder); + } + } catch (Exception e) { + throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e); } - } catch (Exception e) { - throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e); } } loadSubFolders(dir.getAbsolutePath(), f); @@ -347,17 +397,19 @@ public class NpmPackage { if (!d.startsWith("package")) { d = Utilities.path("package", d); } - NpmPackageFolder folder = this.new NpmPackageFolder(d); - folder.folder = f; - this.folders.put(d, folder); File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json")); if (ij.exists()) { - try { - if (!folder.readIndex(JsonParser.parseObject(ij))) { - indexFolder(folder.getName(), folder); + NpmPackageFolder folder = this.new NpmPackageFolder(d); + folder.folder = f; + this.folders.put(d, folder); + if (!minimalMemory) { + try { + if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) { + indexFolder(folder.getFolderName(), folder); + } + } catch (Exception e) { + throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e); } - } catch (Exception e) { - throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e); } } loadSubFolders(rootPath, f); @@ -465,7 +517,8 @@ public class NpmPackage { private void checkIndexed(String desc) throws IOException { for (NpmPackageFolder folder : folders.values()) { - if (folder.index == null || folder.index.forceArray("files").size() == 0) { + JsonObject index = folder.index(); + if (index == null || index.forceArray("files").size() == 0) { indexFolder(desc, folder); } } @@ -484,14 +537,17 @@ public class NpmPackage { folder.removeFile(n); } String json = indexer.build(); + try { - folder.readIndex(JsonParser.parseObject(json)); + if (!minimalMemory) { + folder.readIndex(JsonParser.parseObject(json), folder.getTypes()); + } if (folder.folder != null) { TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json")); } } catch (Exception e) { TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json")); - throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.name+"/.index.json: "+e.getMessage(), e); + throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e); } } @@ -580,10 +636,11 @@ public class NpmPackage { public List listIndexedResources(List types) throws IOException { List res = new ArrayList(); for (NpmPackageFolder folder : folders.values()) { - if (folder.index != null) { - for (JsonObject fi : folder.index.getJsonObjects("files")) { + JsonObject index = folder.index(); + if (index != null) { + for (JsonObject fi : index.getJsonObjects("files")) { if (Utilities.existsInList(fi.asString("resourceType"), types) || types.isEmpty()) { - res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getName() : folder.folder.getAbsolutePath(), fi)); + res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getFolderName() : folder.folder.getAbsolutePath(), fi)); } } } @@ -653,7 +710,7 @@ public class NpmPackage { public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException { NpmPackageFolder f = folders.get(folder); List matches = new ArrayList<>(); - for (JsonObject file : f.index.getJsonObjects("files")) { + for (JsonObject file : f.index().getJsonObjects("files")) { if (canonical.equals(file.asString("url"))) { if (version != null && version.equals(file.asString("version"))) { return load("package", file.asString("filename")); @@ -877,7 +934,7 @@ public class NpmPackage { public InputStream loadResource(String type, String id) throws IOException { NpmPackageFolder f = folders.get("package"); - JsonArray files = f.index.getJsonArray("files"); + JsonArray files = f.index().getJsonArray("files"); for (JsonElement e : files.getItems()) { JsonObject i = (JsonObject) e; if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) { @@ -893,7 +950,7 @@ public class NpmPackage { f = folders.get("package/example"); } if (f != null) { - JsonArray files = f.index.getJsonArray("files"); + JsonArray files = f.index().getJsonArray("files"); for (JsonElement e : files.getItems()) { JsonObject i = (JsonObject) e; if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) { @@ -910,6 +967,7 @@ public class NpmPackage { } public void save(File directory) throws IOException { + assert !minimalMemory; File dir = new File(Utilities.path(directory.getAbsolutePath(), name())); if (!dir.exists()) { Utilities.createDirectory(dir.getAbsolutePath()); @@ -918,7 +976,7 @@ public class NpmPackage { } for (NpmPackageFolder folder : folders.values()) { - String n = folder.name; + String n = folder.folderName; File pd = new File(Utilities.path(dir.getAbsolutePath(), n)); if (!pd.exists()) { @@ -941,6 +999,7 @@ public class NpmPackage { } public void save(OutputStream stream) throws IOException { + assert !minimalMemory; TarArchiveOutputStream tar; ByteArrayOutputStream OutputStream; BufferedOutputStream bufferedOutputStream; @@ -955,7 +1014,7 @@ public class NpmPackage { for (NpmPackageFolder folder : folders.values()) { - String n = folder.name; + String n = folder.folderName; if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) { n = "package/"+n; } @@ -1043,8 +1102,10 @@ public class NpmPackage { } public void unPack(String dir, boolean withAppend) throws IOException { + assert !minimalMemory; + for (NpmPackageFolder folder : folders.values()) { - String dn = folder.getName(); + String dn = folder.getFolderName(); if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) { dn = dn.substring(8); } @@ -1099,6 +1160,8 @@ public class NpmPackage { } public void addFile(String folderName, String name, byte[] cnt, String type) { + assert !minimalMemory; + if (!folders.containsKey(folderName)) { folders.put(folderName, new NpmPackageFolder(folderName)); } @@ -1154,7 +1217,7 @@ public class NpmPackage { return npm.asString("name").startsWith("hl7.terminology"); } - public boolean hasCanonical(String url) { + public boolean hasCanonical(String url) throws IOException { if (url == null) { return false; } @@ -1162,7 +1225,7 @@ public class NpmPackage { String v = url.contains("|") ? url.substring(url.indexOf("|")+1) : null; NpmPackageFolder folder = folders.get("package"); if (folder != null) { - for (JsonObject o : folder.index.getJsonObjects("files")) { + for (JsonObject o : folder.index().getJsonObjects("files")) { if (u.equals(o.asString("url"))) { if (v == null || v.equals(o.asString("version"))) { return true; diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java index 15fb78980..b019cec33 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java @@ -4,11 +4,13 @@ import java.io.File; import java.io.IOException; import java.util.List; +import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.npm.CommonPackages; import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode; import org.hl7.fhir.utilities.npm.NpmPackage; +import org.hl7.fhir.utilities.npm.NpmPackage.NpmPackageFolder; import org.hl7.fhir.utilities.npm.ToolsVersion; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -66,4 +68,23 @@ public class PackageCacheTests { Assertions.assertEquals("0.0.8", cache.loadPackage(CommonPackages.ID_PUBPACK, "0.0.8").version()); Assertions.assertEquals(CommonPackages.VER_PUBPACK, cache.loadPackage(CommonPackages.ID_PUBPACK).version()); } + + @Test + public void testMinimal() throws IOException { + FilesystemPackageCacheManager cache = new FilesystemPackageCacheManager(FilesystemPackageCacheMode.TESTING); + cache.clear(); + NpmPackage uscore = cache.loadPackage("hl7.fhir.us.core", "3.1.0"); + cache.setMinimalMemory(true); + NpmPackage uscoreMin = cache.loadPackage("hl7.fhir.us.core", "3.1.0"); + Assertions.assertEquals(uscore.version(), uscoreMin.version()); + Assertions.assertEquals(uscore.getFolders().size(), uscoreMin.getFolders().size()); + Assertions.assertEquals(uscore.list("package").size(), uscoreMin.list("package").size()); + byte[] b1 = TextFile.streamToBytes(uscore.load(uscore.list("package").get(1))); + byte[] b2 = TextFile.streamToBytes(uscoreMin.load(uscore.list("package").get(1))); +// TextFile.bytesToFile(b1, "/Users/grahamegrieve/temp/b1.json"); +// TextFile.bytesToFile(b2, "/Users/grahamegrieve/temp/b2.json"); + Assertions.assertArrayEquals(b1, b2); + + + } } \ No newline at end of file