Minimal Memory model for NpmPackage
This commit is contained in:
parent
74c30d9143
commit
0615f2a163
|
@ -197,7 +197,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
|
|||
}
|
||||
|
||||
private NpmPackage loadPackageInfo(String path) throws IOException {
|
||||
NpmPackage pi = NpmPackage.fromFolder(path);
|
||||
NpmPackage pi = minimalMemory ? NpmPackage.fromFolderMinimal(path) : NpmPackage.fromFolder(path);
|
||||
return pi;
|
||||
}
|
||||
|
||||
|
|
|
@ -68,6 +68,7 @@ import org.hl7.fhir.utilities.SimpleHTTPClient;
|
|||
import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
|
||||
import org.hl7.fhir.utilities.TextFile;
|
||||
import org.hl7.fhir.utilities.Utilities;
|
||||
import org.hl7.fhir.utilities.json.JsonException;
|
||||
import org.hl7.fhir.utilities.json.model.JsonArray;
|
||||
import org.hl7.fhir.utilities.json.model.JsonElement;
|
||||
import org.hl7.fhir.utilities.json.model.JsonObject;
|
||||
|
@ -162,36 +163,52 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public class NpmPackageFolder {
|
||||
private String name;
|
||||
private Map<String, List<String>> types = new HashMap<>();
|
||||
private Map<String, byte[]> content = new HashMap<>();
|
||||
private JsonObject index;
|
||||
private String folderName;
|
||||
private Map<String, List<String>> types;
|
||||
private Map<String, byte[]> content;
|
||||
private JsonObject cachedIndex;
|
||||
private File folder;
|
||||
|
||||
public NpmPackageFolder(String name) {
|
||||
public NpmPackageFolder(String folderName) {
|
||||
super();
|
||||
this.name = name;
|
||||
this.folderName = folderName;
|
||||
if (!minimalMemory) {
|
||||
types = new HashMap<>();
|
||||
content = new HashMap<>();
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, List<String>> getTypes() {
|
||||
private String fn(String name) throws IOException {
|
||||
return Utilities.path(folder.getAbsolutePath(), name);
|
||||
}
|
||||
|
||||
public Map<String, List<String>> getTypes() throws JsonException, IOException {
|
||||
if (minimalMemory) {
|
||||
Map<String, List<String>> typeMap = new HashMap<>();
|
||||
readIndex(JsonParser.parseObjectFromFile(fn(".index.json")), typeMap);
|
||||
return typeMap;
|
||||
} else {
|
||||
return types;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public boolean readIndex(JsonObject index) {
|
||||
public String getFolderName() {
|
||||
return folderName;
|
||||
}
|
||||
|
||||
public boolean readIndex(JsonObject index, Map<String, List<String>> typeMap) {
|
||||
if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) {
|
||||
return false;
|
||||
}
|
||||
this.index = index;
|
||||
if (!minimalMemory) {
|
||||
this.cachedIndex = index;
|
||||
}
|
||||
for (JsonObject file : index.getJsonObjects("files")) {
|
||||
String type = file.asString("resourceType");
|
||||
String name = file.asString("filename");
|
||||
if (!types.containsKey(type))
|
||||
types.put(type, new ArrayList<>());
|
||||
types.get(type).add(name);
|
||||
if (!typeMap.containsKey(type))
|
||||
typeMap.put(type, new ArrayList<>());
|
||||
typeMap.get(type).add(name);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
@ -216,6 +233,8 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public Map<String, byte[]> getContent() {
|
||||
assert !minimalMemory;
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
|
@ -242,7 +261,7 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public String dump() {
|
||||
return name + " ("+ (folder == null ? "null" : folder.toString())+") | "+Boolean.toString(index != null)+" | "+content.size()+" | "+types.size();
|
||||
return folderName + " ("+ (folder == null ? "null" : folder.toString())+")"+(minimalMemory ? "" : " | "+Boolean.toString(cachedIndex != null)+" | "+content.size()+" | "+types.size());
|
||||
}
|
||||
|
||||
public void removeFile(String n) throws IOException {
|
||||
|
@ -254,13 +273,28 @@ public class NpmPackage {
|
|||
changedByLoader = true;
|
||||
}
|
||||
|
||||
public JsonObject index() throws IOException {
|
||||
if (cachedIndex != null) {
|
||||
return cachedIndex;
|
||||
} else if (folder == null) {
|
||||
return null;
|
||||
} else {
|
||||
File ij = new File(fn(".index.json"));
|
||||
if (ij.exists()) {
|
||||
return JsonParser.parseObject(ij);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String path;
|
||||
private JsonObject npm;
|
||||
private Map<String, NpmPackageFolder> folders = new HashMap<>();
|
||||
private boolean changedByLoader; // internal qa only!
|
||||
private Map<String, Object> userData = new HashMap<>();
|
||||
private Map<String, Object> userData;
|
||||
private boolean minimalMemory;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
|
@ -279,6 +313,17 @@ public class NpmPackage {
|
|||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory method that parses a package from an extracted folder
|
||||
*/
|
||||
public static NpmPackage fromFolderMinimal(String path) throws IOException {
|
||||
NpmPackage res = new NpmPackage();
|
||||
res.minimalMemory = true;
|
||||
res.loadFiles(path, new File(path));
|
||||
res.checkIndexed(path);
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory method that starts a new empty package using the given PackageGenerator to create the manifest
|
||||
*/
|
||||
|
@ -297,6 +342,9 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public Map<String, Object> getUserData() {
|
||||
if (userData == null) {
|
||||
userData = new HashMap<>();
|
||||
}
|
||||
return userData;
|
||||
}
|
||||
|
||||
|
@ -312,19 +360,21 @@ public class NpmPackage {
|
|||
if (!d.equals("package")) {
|
||||
d = Utilities.path("package", d);
|
||||
}
|
||||
File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
|
||||
if (ij.exists()) {
|
||||
NpmPackageFolder folder = this.new NpmPackageFolder(d);
|
||||
folder.folder = f;
|
||||
this.folders.put(d, folder);
|
||||
File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
|
||||
if (ij.exists()) {
|
||||
if (!minimalMemory) {
|
||||
try {
|
||||
if (!folder.readIndex(JsonParser.parseObject(ij))) {
|
||||
indexFolder(folder.getName(), folder);
|
||||
if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
|
||||
indexFolder(folder.getFolderName(), folder);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
loadSubFolders(dir.getAbsolutePath(), f);
|
||||
} else {
|
||||
NpmPackageFolder folder = this.new NpmPackageFolder(Utilities.path("package", "$root"));
|
||||
|
@ -347,19 +397,21 @@ public class NpmPackage {
|
|||
if (!d.startsWith("package")) {
|
||||
d = Utilities.path("package", d);
|
||||
}
|
||||
File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
|
||||
if (ij.exists()) {
|
||||
NpmPackageFolder folder = this.new NpmPackageFolder(d);
|
||||
folder.folder = f;
|
||||
this.folders.put(d, folder);
|
||||
File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
|
||||
if (ij.exists()) {
|
||||
if (!minimalMemory) {
|
||||
try {
|
||||
if (!folder.readIndex(JsonParser.parseObject(ij))) {
|
||||
indexFolder(folder.getName(), folder);
|
||||
if (!folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
|
||||
indexFolder(folder.getFolderName(), folder);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
loadSubFolders(rootPath, f);
|
||||
}
|
||||
}
|
||||
|
@ -465,7 +517,8 @@ public class NpmPackage {
|
|||
|
||||
private void checkIndexed(String desc) throws IOException {
|
||||
for (NpmPackageFolder folder : folders.values()) {
|
||||
if (folder.index == null || folder.index.forceArray("files").size() == 0) {
|
||||
JsonObject index = folder.index();
|
||||
if (index == null || index.forceArray("files").size() == 0) {
|
||||
indexFolder(desc, folder);
|
||||
}
|
||||
}
|
||||
|
@ -484,14 +537,17 @@ public class NpmPackage {
|
|||
folder.removeFile(n);
|
||||
}
|
||||
String json = indexer.build();
|
||||
|
||||
try {
|
||||
folder.readIndex(JsonParser.parseObject(json));
|
||||
if (!minimalMemory) {
|
||||
folder.readIndex(JsonParser.parseObject(json), folder.getTypes());
|
||||
}
|
||||
if (folder.folder != null) {
|
||||
TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
|
||||
throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.name+"/.index.json: "+e.getMessage(), e);
|
||||
throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -580,10 +636,11 @@ public class NpmPackage {
|
|||
public List<PackageResourceInformation> listIndexedResources(List<String> types) throws IOException {
|
||||
List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
|
||||
for (NpmPackageFolder folder : folders.values()) {
|
||||
if (folder.index != null) {
|
||||
for (JsonObject fi : folder.index.getJsonObjects("files")) {
|
||||
JsonObject index = folder.index();
|
||||
if (index != null) {
|
||||
for (JsonObject fi : index.getJsonObjects("files")) {
|
||||
if (Utilities.existsInList(fi.asString("resourceType"), types) || types.isEmpty()) {
|
||||
res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getName() : folder.folder.getAbsolutePath(), fi));
|
||||
res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getFolderName() : folder.folder.getAbsolutePath(), fi));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -653,7 +710,7 @@ public class NpmPackage {
|
|||
public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
|
||||
NpmPackageFolder f = folders.get(folder);
|
||||
List<JsonObject> matches = new ArrayList<>();
|
||||
for (JsonObject file : f.index.getJsonObjects("files")) {
|
||||
for (JsonObject file : f.index().getJsonObjects("files")) {
|
||||
if (canonical.equals(file.asString("url"))) {
|
||||
if (version != null && version.equals(file.asString("version"))) {
|
||||
return load("package", file.asString("filename"));
|
||||
|
@ -877,7 +934,7 @@ public class NpmPackage {
|
|||
|
||||
public InputStream loadResource(String type, String id) throws IOException {
|
||||
NpmPackageFolder f = folders.get("package");
|
||||
JsonArray files = f.index.getJsonArray("files");
|
||||
JsonArray files = f.index().getJsonArray("files");
|
||||
for (JsonElement e : files.getItems()) {
|
||||
JsonObject i = (JsonObject) e;
|
||||
if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
|
||||
|
@ -893,7 +950,7 @@ public class NpmPackage {
|
|||
f = folders.get("package/example");
|
||||
}
|
||||
if (f != null) {
|
||||
JsonArray files = f.index.getJsonArray("files");
|
||||
JsonArray files = f.index().getJsonArray("files");
|
||||
for (JsonElement e : files.getItems()) {
|
||||
JsonObject i = (JsonObject) e;
|
||||
if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
|
||||
|
@ -910,6 +967,7 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public void save(File directory) throws IOException {
|
||||
assert !minimalMemory;
|
||||
File dir = new File(Utilities.path(directory.getAbsolutePath(), name()));
|
||||
if (!dir.exists()) {
|
||||
Utilities.createDirectory(dir.getAbsolutePath());
|
||||
|
@ -918,7 +976,7 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
for (NpmPackageFolder folder : folders.values()) {
|
||||
String n = folder.name;
|
||||
String n = folder.folderName;
|
||||
|
||||
File pd = new File(Utilities.path(dir.getAbsolutePath(), n));
|
||||
if (!pd.exists()) {
|
||||
|
@ -941,6 +999,7 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public void save(OutputStream stream) throws IOException {
|
||||
assert !minimalMemory;
|
||||
TarArchiveOutputStream tar;
|
||||
ByteArrayOutputStream OutputStream;
|
||||
BufferedOutputStream bufferedOutputStream;
|
||||
|
@ -955,7 +1014,7 @@ public class NpmPackage {
|
|||
|
||||
|
||||
for (NpmPackageFolder folder : folders.values()) {
|
||||
String n = folder.name;
|
||||
String n = folder.folderName;
|
||||
if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
|
||||
n = "package/"+n;
|
||||
}
|
||||
|
@ -1043,8 +1102,10 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public void unPack(String dir, boolean withAppend) throws IOException {
|
||||
assert !minimalMemory;
|
||||
|
||||
for (NpmPackageFolder folder : folders.values()) {
|
||||
String dn = folder.getName();
|
||||
String dn = folder.getFolderName();
|
||||
if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
|
||||
dn = dn.substring(8);
|
||||
}
|
||||
|
@ -1099,6 +1160,8 @@ public class NpmPackage {
|
|||
}
|
||||
|
||||
public void addFile(String folderName, String name, byte[] cnt, String type) {
|
||||
assert !minimalMemory;
|
||||
|
||||
if (!folders.containsKey(folderName)) {
|
||||
folders.put(folderName, new NpmPackageFolder(folderName));
|
||||
}
|
||||
|
@ -1154,7 +1217,7 @@ public class NpmPackage {
|
|||
return npm.asString("name").startsWith("hl7.terminology");
|
||||
}
|
||||
|
||||
public boolean hasCanonical(String url) {
|
||||
public boolean hasCanonical(String url) throws IOException {
|
||||
if (url == null) {
|
||||
return false;
|
||||
}
|
||||
|
@ -1162,7 +1225,7 @@ public class NpmPackage {
|
|||
String v = url.contains("|") ? url.substring(url.indexOf("|")+1) : null;
|
||||
NpmPackageFolder folder = folders.get("package");
|
||||
if (folder != null) {
|
||||
for (JsonObject o : folder.index.getJsonObjects("files")) {
|
||||
for (JsonObject o : folder.index().getJsonObjects("files")) {
|
||||
if (u.equals(o.asString("url"))) {
|
||||
if (v == null || v.equals(o.asString("version"))) {
|
||||
return true;
|
||||
|
|
|
@ -4,11 +4,13 @@ import java.io.File;
|
|||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.hl7.fhir.utilities.TextFile;
|
||||
import org.hl7.fhir.utilities.Utilities;
|
||||
import org.hl7.fhir.utilities.npm.CommonPackages;
|
||||
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
|
||||
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode;
|
||||
import org.hl7.fhir.utilities.npm.NpmPackage;
|
||||
import org.hl7.fhir.utilities.npm.NpmPackage.NpmPackageFolder;
|
||||
import org.hl7.fhir.utilities.npm.ToolsVersion;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
@ -66,4 +68,23 @@ public class PackageCacheTests {
|
|||
Assertions.assertEquals("0.0.8", cache.loadPackage(CommonPackages.ID_PUBPACK, "0.0.8").version());
|
||||
Assertions.assertEquals(CommonPackages.VER_PUBPACK, cache.loadPackage(CommonPackages.ID_PUBPACK).version());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMinimal() throws IOException {
|
||||
FilesystemPackageCacheManager cache = new FilesystemPackageCacheManager(FilesystemPackageCacheMode.TESTING);
|
||||
cache.clear();
|
||||
NpmPackage uscore = cache.loadPackage("hl7.fhir.us.core", "3.1.0");
|
||||
cache.setMinimalMemory(true);
|
||||
NpmPackage uscoreMin = cache.loadPackage("hl7.fhir.us.core", "3.1.0");
|
||||
Assertions.assertEquals(uscore.version(), uscoreMin.version());
|
||||
Assertions.assertEquals(uscore.getFolders().size(), uscoreMin.getFolders().size());
|
||||
Assertions.assertEquals(uscore.list("package").size(), uscoreMin.list("package").size());
|
||||
byte[] b1 = TextFile.streamToBytes(uscore.load(uscore.list("package").get(1)));
|
||||
byte[] b2 = TextFile.streamToBytes(uscoreMin.load(uscore.list("package").get(1)));
|
||||
// TextFile.bytesToFile(b1, "/Users/grahamegrieve/temp/b1.json");
|
||||
// TextFile.bytesToFile(b2, "/Users/grahamegrieve/temp/b2.json");
|
||||
Assertions.assertArrayEquals(b1, b2);
|
||||
|
||||
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue