From 444b6128059ff45fabee1c45fa58cc428232c2e7 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Sun, 24 May 2020 15:02:37 -0400 Subject: [PATCH 01/13] Start reworking NPM manager --- .../ExtensionDefinitionGenerator.java | 60 +- .../fhir/r4/test/utils/TestingUtilities.java | 64 +- .../hl7/fhir/r4/test/CDARoundTripTests.java | 4 +- .../r4/test/FHIRMappingLanguageTests.java | 4 +- .../fhir/r5/test/utils/TestingUtilities.java | 63 +- .../hl7/fhir/r5/test/CDARoundTripTests.java | 4 +- .../r5/test/FHIRMappingLanguageTests.java | 4 +- .../fhir/r5/test/SnapShotGenerationTests.java | 4 +- .../r5/test/StructureMapUtilitiesTest.java | 4 +- .../cache/BasePackageCacheManager.java | 60 ++ ...ava => FilesystemPackageCacheManager.java} | 666 +++++++++--------- .../utilities/cache/IPackageCacheManager.java | 15 + .../hl7/fhir/utilities/cache/NpmPackage.java | 63 +- .../utilities/tests/PackageCacheTests.java | 5 +- .../hl7/fhir/validation/ValidationEngine.java | 76 +- .../fhir/validation/cli/utils/Display.java | 4 +- .../validation/packages/PackageValidator.java | 9 +- .../comparison/tests/ComparisonTests.java | 30 +- .../conversion/tests/R3R4ConversionTests.java | 6 +- .../conversion/tests/TestingUtilitiesX.java | 62 +- .../tests/ProfileComparisonTests.java | 13 - 21 files changed, 608 insertions(+), 612 deletions(-) create mode 100644 org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java rename org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/{PackageCacheManager.java => FilesystemPackageCacheManager.java} (77%) create mode 100644 org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java diff --git a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/ExtensionDefinitionGenerator.java b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/ExtensionDefinitionGenerator.java index f36205fd2..66af528f4 100644 --- a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/ExtensionDefinitionGenerator.java +++ b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/ExtensionDefinitionGenerator.java @@ -1,33 +1,33 @@ package org.hl7.fhir.convertors; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ import java.io.IOException; @@ -62,7 +62,7 @@ import org.hl7.fhir.r4.utils.NPMPackageGenerator; import org.hl7.fhir.r4.utils.NPMPackageGenerator.Category; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.cache.NpmPackage; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.PackageGenerator.PackageType; import org.hl7.fhir.utilities.cache.ToolsVersion; @@ -392,7 +392,7 @@ public class ExtensionDefinitionGenerator { private List loadSource() throws IOException, FHIRException { List list = new ArrayList<>(); - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); NpmPackage npm = pcm.loadPackage("hl7.fhir.core", sourceVersion.toCode()); if (sourceVersion == FHIRVersion._4_0_0) context = SimpleWorkerContext.fromPackage(npm); diff --git a/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/test/utils/TestingUtilities.java b/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/test/utils/TestingUtilities.java index cf614edb7..019e639df 100644 --- a/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/test/utils/TestingUtilities.java +++ b/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/test/utils/TestingUtilities.java @@ -1,33 +1,33 @@ package org.hl7.fhir.r4.test.utils; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ import java.io.File; @@ -44,15 +44,13 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.codec.binary.Base64; import org.fhir.ucum.UcumEssenceService; -import org.fhir.ucum.UcumException; -import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.r4.context.IWorkerContext; import org.hl7.fhir.r4.context.SimpleWorkerContext; import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.utilities.CSFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -73,9 +71,9 @@ public class TestingUtilities { public static IWorkerContext context() { if (fcontext == null) { - PackageCacheManager pcm; + FilesystemPackageCacheManager pcm; try { - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); fcontext = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.0")); fcontext.setUcumService(new UcumEssenceService(TestingUtilities.resourceNameToFile("ucum", "ucum-essence.xml"))); fcontext.setExpansionProfile(new Parameters()); diff --git a/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/CDARoundTripTests.java b/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/CDARoundTripTests.java index cf9c320a0..ca1be1254 100644 --- a/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/CDARoundTripTests.java +++ b/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/CDARoundTripTests.java @@ -8,7 +8,7 @@ import org.hl7.fhir.r4.elementmodel.Element; import org.hl7.fhir.r4.elementmodel.Manager; import org.hl7.fhir.r4.elementmodel.Manager.FhirFormat; import org.hl7.fhir.r4.formats.IParser.OutputStyle; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; @@ -29,7 +29,7 @@ public class CDARoundTripTests { @BeforeAll public void setUp() throws Exception { context = new SimpleWorkerContext(); - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); context.loadFromPackage(pcm.loadPackage("hl7.fhir.core", "current"), null, "StructureDefinition"); context.loadFromPackage(pcm.loadPackage("hl7.fhir.cda", "current"), null, "StructureDefinition"); } diff --git a/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/FHIRMappingLanguageTests.java b/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/FHIRMappingLanguageTests.java index 3e65f1afe..5ba09a2a7 100644 --- a/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/FHIRMappingLanguageTests.java +++ b/org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/FHIRMappingLanguageTests.java @@ -14,7 +14,7 @@ import org.hl7.fhir.r4.utils.StructureMapUtilities; import org.hl7.fhir.r4.utils.StructureMapUtilities.ITransformerServices; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.hl7.fhir.utilities.xml.XMLUtil; import org.junit.jupiter.api.BeforeAll; @@ -61,7 +61,7 @@ public class FHIRMappingLanguageTests implements ITransformerServices { @BeforeAll static public void setUp() throws Exception { if (context == null) { - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.0")); jsonParser = new JsonParser(); jsonParser.setOutputStyle(OutputStyle.PRETTY); diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java index 4ab0bb891..064edc59b 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java @@ -19,46 +19,45 @@ import org.fhir.ucum.UcumEssenceService; import org.hl7.fhir.r5.context.IWorkerContext; import org.hl7.fhir.r5.context.SimpleWorkerContext; import org.hl7.fhir.r5.model.Parameters; -import org.hl7.fhir.r5.terminologies.TerminologyClientR5; import org.hl7.fhir.utilities.CSFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.VersionUtilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ import com.google.gson.JsonArray; @@ -83,9 +82,9 @@ public class TestingUtilities { fcontexts = new HashMap<>(); } if (!fcontexts.containsKey(v)) { - PackageCacheManager pcm; + FilesystemPackageCacheManager pcm; try { - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); IWorkerContext fcontext = SimpleWorkerContext.fromPackage(pcm.loadPackage(VersionUtilities.packageForVersion(version), version)); fcontext.setUcumService(new UcumEssenceService(TestingUtilities.loadTestResourceStream("ucum", "ucum-essence.xml"))); fcontext.setExpansionProfile(new Parameters()); diff --git a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CDARoundTripTests.java b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CDARoundTripTests.java index 54e0bcaa1..510b7eeca 100644 --- a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CDARoundTripTests.java +++ b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CDARoundTripTests.java @@ -9,7 +9,7 @@ import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; import org.hl7.fhir.r5.model.StructureDefinition; import org.hl7.fhir.r5.test.utils.TestingUtilities; import org.hl7.fhir.r5.utils.FHIRPathEngine; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -176,7 +176,7 @@ public class CDARoundTripTests { @Test @Disabled public void testSimple() throws IOException { - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); SimpleWorkerContext context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r4.core", "4.0.1")); context.loadFromFile(TestingUtilities.loadTestResourceStream("r5", "cda", "any.xml"), "any.xml", null); context.loadFromFile(TestingUtilities.loadTestResourceStream("r5", "cda", "ii.xml"), "ii.xml", null); diff --git a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/FHIRMappingLanguageTests.java b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/FHIRMappingLanguageTests.java index b453cdeb6..ad2fbde39 100644 --- a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/FHIRMappingLanguageTests.java +++ b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/FHIRMappingLanguageTests.java @@ -33,7 +33,7 @@ import org.hl7.fhir.r5.utils.StructureMapUtilities; import org.hl7.fhir.r5.utils.StructureMapUtilities.ITransformerServices; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.hl7.fhir.utilities.xml.XMLUtil; import org.junit.jupiter.api.BeforeAll; @@ -67,7 +67,7 @@ public class FHIRMappingLanguageTests implements ITransformerServices { @BeforeAll public static void setUp() throws Exception { - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.1")); jsonParser = new JsonParser(); jsonParser.setOutputStyle(OutputStyle.PRETTY); diff --git a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/SnapShotGenerationTests.java b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/SnapShotGenerationTests.java index 5c95a28c6..c092132ec 100644 --- a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/SnapShotGenerationTests.java +++ b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/SnapShotGenerationTests.java @@ -43,7 +43,7 @@ import org.hl7.fhir.r5.utils.IResourceValidator; import org.hl7.fhir.r5.utils.XVerExtensionManager; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.cache.NpmPackage; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; @@ -522,7 +522,7 @@ public class SnapShotGenerationTests { pu.setDebug(test.isDebug()); pu.setIds(test.getSource(), false); if (!TestingUtilities.context().hasPackage("hl7.fhir.xver-extensions", "0.0.4")) { - NpmPackage npm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION).loadPackage("hl7.fhir.xver-extensions", "0.0.4"); + NpmPackage npm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION).loadPackage("hl7.fhir.xver-extensions", "0.0.4"); TestingUtilities.context().loadFromPackage(npm, new TestLoader(new String[]{"StructureDefinition"}), new String[]{"StructureDefinition"}); } pu.setXver(new XVerExtensionManager(TestingUtilities.context())); diff --git a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/StructureMapUtilitiesTest.java b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/StructureMapUtilitiesTest.java index 6a7681850..5fc893042 100644 --- a/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/StructureMapUtilitiesTest.java +++ b/org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/StructureMapUtilitiesTest.java @@ -13,7 +13,7 @@ import org.hl7.fhir.r5.model.StructureMap; import org.hl7.fhir.r5.test.utils.TestingUtilities; import org.hl7.fhir.r5.utils.StructureMapUtilities; import org.hl7.fhir.r5.utils.StructureMapUtilities.ITransformerServices; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -25,7 +25,7 @@ public class StructureMapUtilitiesTest implements ITransformerServices { // @BeforeAll static public void setUp() throws Exception { - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.core", "4.0.0")); } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java new file mode 100644 index 000000000..bc7b25ec7 --- /dev/null +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java @@ -0,0 +1,60 @@ +package org.hl7.fhir.utilities.cache; + +import org.apache.commons.lang3.Validate; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public abstract class BasePackageCacheManager implements IPackageCacheManager { + + private List myPackageServers = new ArrayList<>(); + + public List getPackageServers() { + return myPackageServers; + } + + /** + * Add a package server that can be used to fetch remote packages + */ + public void addPackageServer(@Nonnull String thePackageServer) { + Validate.notBlank(thePackageServer, "thePackageServer must not be null or empty"); + if (!myPackageServers.contains(thePackageServer)) { + myPackageServers.add(thePackageServer); + } + } + + + @Override + public String getPackageId(String canonical) throws IOException { + String result = null; + + for (String nextPackageServer : getPackageServers()) { + result = getPackageId(canonical, nextPackageServer); + if (result != null) { + break; + } + } + + return result; + } + + private String getPackageId(String canonical, String server) throws IOException { + PackageClient pc = new PackageClient(server); + List res = pc.search(null, canonical, null, false); + if (res.size() == 0) { + return null; + } else { + // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others) + for (PackageClient.PackageInfo pi : res) { + if (canonical.equals(pi.getCanonical())) { + return pi.getId(); + } + } + return res.get(0).getId(); + } + } + + +} diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java similarity index 77% rename from org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java rename to org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java index c35172733..7ac21b3c3 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java @@ -1,38 +1,50 @@ package org.hl7.fhir.utilities.cache; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.apache.commons.io.FileUtils; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.utilities.IniFile; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder; +import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; +import org.hl7.fhir.utilities.json.JSONUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -54,154 +66,38 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; -import org.hl7.fhir.exceptions.FHIRException; -import org.hl7.fhir.utilities.IniFile; -import org.hl7.fhir.utilities.TextFile; -import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder; -import org.hl7.fhir.utilities.cache.PackageCacheManager.BuildRecord; -import org.hl7.fhir.utilities.cache.PackageCacheManager.BuildRecordSorter; -import org.hl7.fhir.utilities.cache.PackageCacheManager.InputStreamWithSrc; -import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; -import org.hl7.fhir.utilities.json.JSONUtil; - -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - /** * Package cache manager - * + *

* API: - * + *

* constructor - * getPackageUrl - * getPackageId - * findPackageCache - * addPackageToCache - * - * @author Grahame Grieve + * getPackageUrl + * getPackageId + * findPackageCache + * addPackageToCache * + * @author Grahame Grieve */ -public class PackageCacheManager { - - public class InputStreamWithSrc { - - public InputStream stream; - public String url; - public String version; - - public InputStreamWithSrc(InputStream stream, String url, String version) { - this.stream = stream; - this.url = url; - this.version = version; - } - } - - public class BuildRecordSorter implements Comparator { - - @Override - public int compare(BuildRecord arg0, BuildRecord arg1) { - return arg1.date.compareTo(arg0.date); - } - } - - public class BuildRecord { - - private String url; - private String packageId; - private String repo; - private Date date; - public BuildRecord(String url, String packageId, String repo, Date date) { - super(); - this.url = url; - this.packageId = packageId; - this.repo = repo; - this.date = date; - } - public String getUrl() { - return url; - } - public String getPackageId() { - return packageId; - } - public String getRepo() { - return repo; - } - public Date getDate() { - return date; - } - - - } - - /** if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly. - * - * You can use this interface to - * @author graha - * - */ - public interface INetworkServices { - - InputStream resolvePackage(String packageId, String version); - } - - public class VersionHistory { - private String id; - private String canonical; - private String current; - private Map versions = new HashMap<>(); - public String getCanonical() { - return canonical; - } - public String getCurrent() { - return current; - } - public Map getVersions() { - return versions; - } - public String getId() { - return id; - } - } - - - public class PackageEntry { - - private byte[] bytes; - private String name; - - public PackageEntry(String name) { - this.name = name; - } - - public PackageEntry(String name, byte[] bytes) { - this.name = name; - this.bytes = bytes; - } - } - - public static final String PRIMARY_SERVER = "http://packages.fhir.org"; - public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages"; -// private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages"; +public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager { + private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class); public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$"; public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$"; private static final String CACHE_VERSION = "3"; // second version - see wiki page private String cacheFolder; private boolean progress = true; private List temporaryPackages = new ArrayList(); - -// private List allUrls; -// private Map historyCache = new HashMap<>(); - - // ========================= Initialization ============================================================================ + private boolean buildLoaded = false; + private Map ciList = new HashMap(); + private JsonArray buildInfo; + + /** + * Constructor + */ + public FilesystemPackageCacheManager(boolean userMode, int toolsVersion) throws IOException { + addPackageServer("http://packages.fhir.org"); + addPackageServer("http://packages2.fhir.org/packages"); - public PackageCacheManager(boolean userMode, int toolsVersion) throws IOException { if (userMode) cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages"); else @@ -209,7 +105,7 @@ public class PackageCacheManager { if (!(new File(cacheFolder).exists())) Utilities.createDirectory(cacheFolder); if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists())) - TextFile.stringToFile("[cache]\r\nversion="+CACHE_VERSION+"\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false); + TextFile.stringToFile("[cache]\r\nversion=" + CACHE_VERSION + "\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false); createIniFile(); } @@ -223,16 +119,15 @@ public class PackageCacheManager { } } } - - // ========================= Utilities ============================================================================ - + public String getFolder() { return cacheFolder; } - private static String userDir() throws IOException { - return Utilities.path(System.getProperty("user.home"), ".fhir", "packages"); - } +// private List allUrls; +// private Map historyCache = new HashMap<>(); + + // ========================= Initialization ============================================================================ private List sorted(String[] keys) { List names = new ArrayList(); @@ -247,29 +142,30 @@ public class PackageCacheManager { return pi; } + // ========================= Utilities ============================================================================ + private JsonObject fetchJson(String source) throws IOException { URL url = new URL(source); URLConnection c = url.openConnection(); return (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(c.getInputStream())); } - + private void clearCache() throws IOException { for (File f : new File(cacheFolder).listFiles()) { if (f.isDirectory()) { Utilities.clearDirectory(f.getAbsolutePath()); try { - FileUtils.deleteDirectory(f); + FileUtils.deleteDirectory(f); } catch (Exception e1) { try { FileUtils.deleteDirectory(f); - } catch (Exception e2) { - // just give up - } + } catch (Exception e2) { + // just give up + } } - } - else if (!f.getName().equals("packages.ini")) + } else if (!f.getName().equals("packages.ini")) FileUtils.forceDelete(f); - } + } IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); ini.removeSection("packages"); ini.save(); @@ -278,7 +174,7 @@ public class PackageCacheManager { private void createIniFile() throws IOException { IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); boolean save = false; - String v = ini.getStringProperty("cache", "version"); + String v = ini.getStringProperty("cache", "version"); if (!CACHE_VERSION.equals(v)) { clearCache(); ini.setStringProperty("cache", "version", CACHE_VERSION, null); @@ -288,34 +184,18 @@ public class PackageCacheManager { private void checkValidVersionString(String version, String id) { if (Utilities.noString(version)) { - throw new FHIRException("Cannot add package "+id+" to the package cache - a version must be provided"); + throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided"); } if (version.startsWith("file:")) { - throw new FHIRException("Cannot add package "+id+" to the package cache - the version '"+version+"' is illegal in this context"); + throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context"); } for (char ch : version.toCharArray()) { if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-')) { - throw new FHIRException("Cannot add package "+id+" to the package cache - the version '"+version+"' is illegal (ch '"+ch+"'"); + throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'"); } } } - private String getPackageId(String canonical, String server) throws IOException { - PackageClient pc = new PackageClient(server); - List res = pc.search(null, canonical, null, false); - if (res.size() == 0) { - return null; - } else { - // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others) - for (PackageInfo pi : res) { - if (canonical.equals(pi.getCanonical())) { - return pi.getId(); - } - } - return res.get(0).getId(); - } - } - private String getPackageUrl(String packageId, String server) throws IOException { PackageClient pc = new PackageClient(server); List res = pc.search(packageId, null, null, false); @@ -333,92 +213,81 @@ public class PackageCacheManager { if (!specList.containsKey(m.getId())) { specList.put(m.getId(), m.getUrl()); } - } + } } - private InputStreamWithSrc loadFromPackageServer(String id, String v) { - PackageClient pc = new PackageClient(PRIMARY_SERVER); - String u = null; - InputStream stream; - try { - if (Utilities.noString(v)) { - v = pc.getLatestVersion(id); - } - stream = pc.fetch(id, v); - u = pc.url(id, v); - } catch (IOException e) { - pc = new PackageClient(SECONDARY_SERVER); + private InputStreamWithSrc loadFromPackageServer(String id, String version) { + + for (String nextPackageServer : getPackageServers()) { + PackageClient packageClient = new PackageClient(nextPackageServer); try { - if (Utilities.noString(v)) { - v = pc.getLatestVersion(id); + if (Utilities.noString(version)) { + version = packageClient.getLatestVersion(id); } - stream = pc.fetch(id, v); - u = pc.url(id, v); - } catch (IOException e1) { - // ok, well, we'll try the old way - return fetchTheOldWay(id, v); + InputStream stream = packageClient.fetch(id, version); + String url = packageClient.url(id, version); + return new InputStreamWithSrc(stream, packageClient.url(id, version), version); + } catch (IOException e) { + ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer); } + } - return new InputStreamWithSrc(stream, pc.url(id, v), v); + + // ok, well, we'll try the old way + return fetchTheOldWay(id, version); } public String getLatestVersion(String id) throws IOException { - PackageClient pc = new PackageClient(PRIMARY_SERVER); - try { - return pc.getLatestVersion(id); - } catch (IOException e) { - pc = new PackageClient(SECONDARY_SERVER); + for (String nextPackageServer : getPackageServers()) { + PackageClient pc = new PackageClient(nextPackageServer); try { return pc.getLatestVersion(id); - } catch (IOException e1) { - return fetchVersionTheOldWay(id); + } catch (IOException e) { + ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer); } } + + return fetchVersionTheOldWay(id); } - - private NpmPackage loadPackageFromFile(String id, String folder) throws IOException { File f = new File(Utilities.path(folder, id)); if (!f.exists()) { - throw new FHIRException("Package '"+id+" not found in folder "+folder); + throw new FHIRException("Package '" + id + " not found in folder " + folder); } if (!f.isDirectory()) { - throw new FHIRException("File for '"+id+" found in folder "+folder+", not a folder"); + throw new FHIRException("File for '" + id + " found in folder " + folder + ", not a folder"); } File fp = new File(Utilities.path(folder, id, "package", "package.json")); if (!fp.exists()) { - throw new FHIRException("Package '"+id+" found in folder "+folder+", but does not contain a package.json file in /package"); + throw new FHIRException("Package '" + id + " found in folder " + folder + ", but does not contain a package.json file in /package"); } return NpmPackage.fromFolder(f.getAbsolutePath()); } - - // ========================= Package Mgmt API ======================================================================= - /** * Clear the cache - * + * * @throws IOException */ public void clear() throws IOException { clearCache(); } - + /** * Remove a particular package from the cache - * + * * @param id * @param ver * @throws IOException */ - public void removePackage(String id, String ver) throws IOException { - String f = Utilities.path(cacheFolder, id+"#"+ver); + public void removePackage(String id, String ver) throws IOException { + String f = Utilities.path(cacheFolder, id + "#" + ver); File ff = new File(f); - if (ff.exists()) { - Utilities.clearDirectory(f); + if (ff.exists()) { + Utilities.clearDirectory(f); IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); - ini.removeProperty("packages", id+"#"+ver); + ini.removeProperty("packages", id + "#" + ver); ini.save(); ff.delete(); } @@ -426,21 +295,21 @@ public class PackageCacheManager { /** * Load the latest version of the identified package from the cache - it it exists - * + * * @param id * @return * @throws IOException */ public NpmPackage loadPackageFromCacheOnly(String id) throws IOException { - return loadPackageFromCacheOnly(id, null); + return loadPackageFromCacheOnly(id, null); } - - /** - * Load the identified package from the cache - it it exists - * - * This is for special purpose only (testing, control over speed of loading). + + /** + * Load the identified package from the cache - it it exists + *

+ * This is for special purpose only (testing, control over speed of loading). * Generally, use the loadPackage method - * + * * @param id * @param version * @return @@ -460,8 +329,8 @@ public class PackageCacheManager { } } for (String f : sorted(new File(cacheFolder).list())) { - if (f.equals(id+"#"+version) || (Utilities.noString(version) && f.startsWith(id+"#"))) { - return loadPackageInfo(Utilities.path(cacheFolder, f)); + if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) { + return loadPackageInfo(Utilities.path(cacheFolder, f)); } } if ("dev".equals(version)) @@ -470,31 +339,34 @@ public class PackageCacheManager { return null; } + + // ========================= Package Mgmt API ======================================================================= + /** * Add an already fetched package to the cache */ public NpmPackage addPackageToCache(String id, String version, InputStream tgz, String sourceDesc) throws IOException { checkValidVersionString(version, id); - if (progress ) { - System.out.println("Installing "+id+"#"+(version == null ? "?" : version)+" to the package cache"); + if (progress) { + System.out.println("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache"); System.out.print(" Fetching:"); } NpmPackage npm = NpmPackage.fromPackage(tgz, sourceDesc, true); - if (progress ) { + if (progress) { System.out.println(); System.out.print(" Installing: "); } if (npm.name() == null || id == null || !id.equals(npm.name())) { if (!id.equals("hl7.fhir.r5.core")) {// temporary work around - throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name()); + throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name()); } } if (version == null) version = npm.version(); - String packRoot = Utilities.path(cacheFolder, id+"#"+version); + String packRoot = Utilities.path(cacheFolder, id + "#" + version); try { Utilities.createDirectory(packRoot); Utilities.clearDirectory(packRoot); @@ -503,7 +375,8 @@ public class PackageCacheManager { int c = 0; int size = 0; for (Entry e : npm.getFolders().entrySet()) { - String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());; + String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey()); + ; if (!(new File(dir).exists())) Utilities.createDirectory(dir); for (Entry fe : e.getValue().getContent().entrySet()) { @@ -520,15 +393,15 @@ public class PackageCacheManager { System.out.print(" "); c = 2; } - } + } } } IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); ini.setTimeStampFormat("yyyyMMddhhmmss"); - ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null); - ini.setIntegerProperty("package-sizes", id+"#"+version, size, null); + ini.setTimestampProperty("packages", id + "#" + version, Timestamp.from(Instant.now()), null); + ini.setIntegerProperty("package-sizes", id + "#" + version, size, null); ini.save(); if (progress) System.out.println(" done."); @@ -545,7 +418,7 @@ public class PackageCacheManager { npm.getNpm().remove("version"); npm.getNpm().addProperty("version", version); } - TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false); + TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id + "#" + version, "package", "package.json"), false); } return pck; } catch (Exception e) { @@ -560,38 +433,28 @@ public class PackageCacheManager { } } - public String getPackageId(String canonical) throws IOException { - String result = null; - if (result == null) { - result = getPackageId(canonical, PRIMARY_SERVER); - } - if (result == null) { - result = getPackageId(canonical, SECONDARY_SERVER); - } - if (result == null) { - result = getPackageIdFromBuildList(canonical); - } - return result; - } - + @Override public String getPackageUrl(String packageId) throws IOException { String result = null; NpmPackage npm = loadPackageFromCacheOnly(packageId); if (npm != null) { return npm.canonical(); - } - if (result == null) { - getPackageUrl(packageId, PRIMARY_SERVER); } - if (result == null) { - result = getPackageUrl(packageId, SECONDARY_SERVER); + + for (String nextPackageServer : getPackageServers()) { + result = getPackageUrl(nextPackageServer); + if (result != null) { + return result; + } } + if (result == null) { result = getPackageUrlFromBuildList(packageId); } + return result; } - + public void listAllIds(Map specList) throws IOException { for (NpmPackage p : temporaryPackages) { specList.put(p.name(), p.canonical()); @@ -601,46 +464,40 @@ public class PackageCacheManager { addCIBuildSpecs(specList); } - public NpmPackage loadPackage(String id, String v) throws FHIRException, IOException { + @Override + public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException { //ok, try to resolve locally - if (!Utilities.noString(v) && v.startsWith("file:")) { - return loadPackageFromFile(id, v.substring(5)); + if (!Utilities.noString(version) && version.startsWith("file:")) { + return loadPackageFromFile(id, version.substring(5)); } - NpmPackage p = loadPackageFromCacheOnly(id, v); + NpmPackage p = loadPackageFromCacheOnly(id, version); if (p != null) { - if ("current".equals(v)) { + if ("current".equals(version)) { p = checkCurrency(id, p); } if (p != null) return p; } - if ("dev".equals(v)) { + if ("dev".equals(version)) { p = loadPackageFromCacheOnly(id, "current"); p = checkCurrency(id, p); if (p != null) return p; - v = "current"; + version = "current"; } // nup, don't have it locally (or it's expired) InputStreamWithSrc source; - if ("current".equals(v)) { + if ("current".equals(version)) { // special case - fetch from ci-build server source = loadFromCIBuild(id); } else { - source = loadFromPackageServer(id, v); + source = loadFromPackageServer(id, version); } - return addPackageToCache(id, v == null ? source.version : v, source.stream, source.url); + return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url); } - - // ---------- Current Build SubSystem -------------------------------------------------------------------------------------- - - private boolean buildLoaded = false; - private Map ciList = new HashMap(); - private JsonArray buildInfo; - private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException { try { URL url = new URL(source); @@ -653,39 +510,21 @@ public class PackageCacheManager { throw new FHIRException(e.getMessage(), e); } } - + private InputStreamWithSrc loadFromCIBuild(String id) throws IOException { checkBuildLoaded(); if (ciList.containsKey(id)) { InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false); return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current"); } else if (id.startsWith("hl7.fhir.r5")) { - InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id+".tgz"), false); - return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id+".tgz"), "current"); + InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id + ".tgz"), false); + return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id + ".tgz"), "current"); } else { - throw new FHIRException("The package '"+id+"' has not entry on the current build server"); + throw new FHIRException("The package '" + id + "' has not entry on the current build server"); } } - - private String getPackageIdFromBuildList(String canonical) throws IOException { - checkBuildLoaded(); - if (buildInfo != null) { - for (JsonElement n : buildInfo) { - JsonObject o = (JsonObject) n; - if (canonical.equals(JSONUtil.str(o, "url"))) { - return JSONUtil.str(o, "package-id"); - } - } - for (JsonElement n : buildInfo) { - JsonObject o = (JsonObject) n; - if (JSONUtil.str(o, "url").startsWith(canonical+"/ImplementationGuide/")) { - return JSONUtil.str(o, "package-id"); - } - } - } - return null; - } - + + private String getPackageUrlFromBuildList(String packageId) throws IOException { checkBuildLoaded(); for (JsonElement n : buildInfo) { @@ -698,6 +537,8 @@ public class PackageCacheManager { } + // ---------- Current Build SubSystem -------------------------------------------------------------------------------------- + private void addCIBuildSpecs(Map specList) throws IOException { checkBuildLoaded(); for (JsonElement n : buildInfo) { @@ -705,9 +546,38 @@ public class PackageCacheManager { if (!specList.containsKey(JSONUtil.str(o, "package-id"))) { specList.put(JSONUtil.str(o, "package-id"), JSONUtil.str(o, "url")); } - } + } } + @Override + public String getPackageId(String canonical) throws IOException { + String retVal = super.getPackageId(canonical); + + if (retVal == null) { + retVal = getPackageIdFromBuildList(canonical); + } + + return retVal; + } + + private String getPackageIdFromBuildList(String canonical) throws IOException { + checkBuildLoaded(); + if (buildInfo != null) { + for (JsonElement n : buildInfo) { + JsonObject o = (JsonObject) n; + if (canonical.equals(JSONUtil.str(o, "url"))) { + return JSONUtil.str(o, "package-id"); + } + } + for (JsonElement n : buildInfo) { + JsonObject o = (JsonObject) n; + if (JSONUtil.str(o, "url").startsWith(canonical + "/ImplementationGuide/")) { + return JSONUtil.str(o, "package-id"); + } + } + } + return null; + } private NpmPackage checkCurrency(String id, NpmPackage p) throws IOException { checkBuildLoaded(); @@ -718,34 +588,33 @@ public class PackageCacheManager { String currDate = JSONUtil.str(json, "date"); String packDate = p.date(); if (!currDate.equals(packDate)) - return null; // nup, we need a new copy + return null; // nup, we need a new copy return p; } catch (Exception e) { return p; } } - + private boolean checkBuildLoaded() { if (buildLoaded) return true; try { loadFromBuildServer(); } catch (Exception e) { - System.out.println("Error connecting to build server - running without build ("+e.getMessage()+")"); + System.out.println("Error connecting to build server - running without build (" + e.getMessage() + ")"); } return false; } - - + private void loadFromBuildServer() throws IOException { URL url = new URL("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis()); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); InputStream json = connection.getInputStream(); buildInfo = (JsonArray) new com.google.gson.JsonParser().parse(TextFile.streamToString(json)); - + List builds = new ArrayList<>(); - + for (JsonElement n : buildInfo) { JsonObject o = (JsonObject) n; if (o.has("url") && o.has("package-id") && o.get("package-id").getAsString().contains(".")) { @@ -758,13 +627,13 @@ public class PackageCacheManager { Collections.sort(builds, new BuildRecordSorter()); for (BuildRecord bld : builds) { if (!ciList.containsKey(bld.getPackageId())) { - ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/"+bld.getRepo()); + ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo()); } } buildLoaded = true; // whether it succeeds or not } -// private String buildPath(String url) { + // private String buildPath(String url) { // for (JsonElement e : buildInfo) { // JsonObject j = (JsonObject) e; // if (j.has("url") && (url.equals(j.get("url").getAsString()) || j.get("url").getAsString().startsWith(url+"/ImplementationGuide"))) { @@ -773,12 +642,12 @@ public class PackageCacheManager { // } // return null; // } -// +// private String getRepo(String path) { String[] p = path.split("\\/"); - return p[0]+"/"+p[1]; + return p[0] + "/" + p[1]; } - + private Date readDate(String s) { SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US")); try { @@ -800,7 +669,7 @@ public class PackageCacheManager { } } if (url == null) { - throw new FHIRException("Unable to resolve package id "+id+"#"+v); + throw new FHIRException("Unable to resolve package id " + id + "#" + v); } if (url.contains("/ImplementationGuide/")) { url = url.substring(0, url.indexOf("/ImplementationGuide/")); @@ -817,11 +686,11 @@ public class PackageCacheManager { InputStreamWithSrc src = new InputStreamWithSrc(fetchFromUrlSpecific(pv, true), pv, v); return src; } catch (Exception e1) { - throw new FHIRException("Error fetching package directly ("+pv+"), or fetching package list for "+id+" from "+pu+": "+e1.getMessage(), e1); - } + throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1); + } } if (!id.equals(JSONUtil.str(json, "package-id"))) - throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id")); + throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id")); for (JsonElement e : json.getAsJsonArray("list")) { JsonObject vo = (JsonObject) e; if (v.equals(JSONUtil.str(vo, "version"))) { @@ -844,12 +713,12 @@ public class PackageCacheManager { } } if (url == null) { - throw new FHIRException("Unable to resolve package id "+id); + throw new FHIRException("Unable to resolve package id " + id); } String pu = Utilities.pathURL(url, "package-list.json"); JsonObject json = fetchJson(pu); if (!id.equals(JSONUtil.str(json, "package-id"))) - throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id")); + throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id")); for (JsonElement e : json.getAsJsonArray("list")) { JsonObject vo = (JsonObject) e; if (JSONUtil.bool(vo, "current")) { @@ -878,6 +747,115 @@ public class PackageCacheManager { } + /** + * if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly. + *

+ * You can use this interface to + * + * @author graha + */ + public interface INetworkServices { + + InputStream resolvePackage(String packageId, String version); + } + + public class InputStreamWithSrc { + + public InputStream stream; + public String url; + public String version; + + public InputStreamWithSrc(InputStream stream, String url, String version) { + this.stream = stream; + this.url = url; + this.version = version; + } + } + + public class BuildRecordSorter implements Comparator { + + @Override + public int compare(BuildRecord arg0, BuildRecord arg1) { + return arg1.date.compareTo(arg0.date); + } + } + + public class BuildRecord { + + private String url; + private String packageId; + private String repo; + private Date date; + + public BuildRecord(String url, String packageId, String repo, Date date) { + super(); + this.url = url; + this.packageId = packageId; + this.repo = repo; + this.date = date; + } + + public String getUrl() { + return url; + } + + public String getPackageId() { + return packageId; + } + + public String getRepo() { + return repo; + } + + public Date getDate() { + return date; + } + + + } + + public class VersionHistory { + private String id; + private String canonical; + private String current; + private Map versions = new HashMap<>(); + + public String getCanonical() { + return canonical; + } + + public String getCurrent() { + return current; + } + + public Map getVersions() { + return versions; + } + + public String getId() { + return id; + } + } + + public class PackageEntry { + + private byte[] bytes; + private String name; + + public PackageEntry(String name) { + this.name = name; + } + + public PackageEntry(String name, byte[] bytes) { + this.name = name; + this.bytes = bytes; + } + } + + private static String userDir() throws IOException { + return Utilities.path(System.getProperty("user.home"), ".fhir", "packages"); + } + //public List getUrls() throws IOException { // if (allUrls == null) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java new file mode 100644 index 000000000..40a6e61c7 --- /dev/null +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java @@ -0,0 +1,15 @@ +package org.hl7.fhir.utilities.cache; + +import org.hl7.fhir.exceptions.FHIRException; + +import java.io.IOException; + +public interface IPackageCacheManager { + + + String getPackageId(String canonical) throws IOException; + + String getPackageUrl(String packageId) throws IOException; + + NpmPackage loadPackage(String id, String version) throws FHIRException, IOException; +} diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java index 8243a068e..e2b047ba1 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java @@ -1,33 +1,33 @@ package org.hl7.fhir.utilities.cache; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ @@ -35,7 +35,6 @@ import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -57,20 +56,14 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; -import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.cache.NpmPackage.IndexVersionSorter; -import org.hl7.fhir.utilities.cache.PackageCacheManager.PackageEntry; import org.hl7.fhir.utilities.cache.PackageGenerator.PackageType; import org.hl7.fhir.utilities.json.JSONUtil; import org.hl7.fhir.utilities.json.JsonTrackingParser; -import com.google.common.base.Charsets; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java index 46735d07f..6bc7813a9 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageCacheTests.java @@ -2,10 +2,9 @@ package org.hl7.fhir.utilities.tests; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.cache.NpmPackage; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.File; @@ -15,7 +14,7 @@ public class PackageCacheTests { @Test public void testPath() throws IOException { - PackageCacheManager cache = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager cache = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); cache.clear(); Assertions.assertTrue(cache.listPackages().isEmpty()); NpmPackage npm = cache.loadPackage("hl7.fhir.pubpack", "0.0.3"); diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java index 4209ba440..264963fc6 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java @@ -4,7 +4,6 @@ import org.apache.commons.io.IOUtils; import org.hl7.fhir.convertors.*; import org.hl7.fhir.exceptions.DefinitionException; import org.hl7.fhir.exceptions.FHIRException; -import org.hl7.fhir.exceptions.FHIRFormatError; import org.hl7.fhir.r5.conformance.ProfileUtilities; import org.hl7.fhir.r5.context.IWorkerContext.IContextResourceLoader; import org.hl7.fhir.r5.context.IWorkerContext.PackageVersion; @@ -33,7 +32,7 @@ import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.VersionUtilities; import org.hl7.fhir.utilities.cache.NpmPackage; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; @@ -42,11 +41,8 @@ import org.hl7.fhir.utilities.validation.ValidationMessage.Source; import org.hl7.fhir.utilities.xhtml.XhtmlComposer; import org.xml.sax.SAXException; -import com.google.gson.JsonObject; - import java.io.*; import java.net.HttpURLConnection; -import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; @@ -55,34 +51,34 @@ import java.util.Map.Entry; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ /* Copyright (c) 2011+, HL7, Inc @@ -254,7 +250,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { private boolean anyExtensionsAllowed = false; private String version; private String language; - private PackageCacheManager pcm; + private FilesystemPackageCacheManager pcm; private PrintWriter mapLog; private boolean debug; private Set loadedIgs = new HashSet<>(); @@ -301,7 +297,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { } public ValidationEngine() throws IOException { - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); } public void setTerminologyServer(String src, String log, FhirPublication version) throws Exception { @@ -325,7 +321,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { } public ValidationEngine(String src, String txsrvr, String txLog, FhirPublication version, boolean canRunWithoutTerminologyServer, String vString) throws Exception { - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); loadCoreDefinitions(src, false); context.setCanRunWithoutTerminology(canRunWithoutTerminologyServer); setTerminologyServer(txsrvr, txLog, version); @@ -333,7 +329,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { } public ValidationEngine(String src, String txsrvr, String txLog, FhirPublication version, String vString) throws Exception { - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); loadCoreDefinitions(src, false); setTerminologyServer(txsrvr, txLog, version); this.version = vString; @@ -341,7 +337,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { public ValidationEngine(String src) throws Exception { loadCoreDefinitions(src, false); - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); } public String getLanguage() { @@ -473,7 +469,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { res.put(Utilities.changeFileExt(src, "."+fmt.getExtension()), TextFile.fileToBytesNCS(src)); return res; } - } else if ((src.matches(PackageCacheManager.PACKAGE_REGEX) || src.matches(PackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) { + } else if ((src.matches(FilesystemPackageCacheManager.PACKAGE_REGEX) || src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) { return fetchByPackage(src); } throw new Exception("Unable to find/resolve/read -ig "+src); @@ -627,7 +623,7 @@ public class ValidationEngine implements IValidatorResourceFetcher { } if (pcm == null) { log("Creating Package manager?"); - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); } if (version == null) { version = pcm.getLatestVersion(id); diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Display.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Display.java index da1b1ea5d..fb8b08156 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Display.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Display.java @@ -2,7 +2,7 @@ package org.hl7.fhir.validation.cli.utils; import org.hl7.fhir.r5.model.Constants; import org.hl7.fhir.utilities.VersionUtilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import java.io.IOException; @@ -20,7 +20,7 @@ public class Display { System.out.print(s.contains(" ") ? " \"" + s + "\"" : " " + s); } System.out.println(); - System.out.println("Directories: Current = " + System.getProperty("user.dir") + ", Package Cache = " + new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION).getFolder()); + System.out.println("Directories: Current = " + System.getProperty("user.dir") + ", Package Cache = " + new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION).getFolder()); } public static void displayHelpDetails() { diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java index 42d5df892..3837bcf7f 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java @@ -6,14 +6,11 @@ import java.util.List; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.exceptions.FHIRFormatError; -import org.hl7.fhir.r5.context.IWorkerContext; -import org.hl7.fhir.r5.context.SimpleWorkerContext; import org.hl7.fhir.utilities.VersionUtilities; import org.hl7.fhir.utilities.cache.NpmPackage; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.PackageClient; import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; -import org.xml.sax.InputSource; import org.hl7.fhir.utilities.cache.ToolsVersion; public class PackageValidator { @@ -23,9 +20,9 @@ public class PackageValidator { } private void execute() throws IOException { - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); - PackageClient pc = new PackageClient(PackageCacheManager.PRIMARY_SERVER); + PackageClient pc = new PackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER); for (PackageInfo t : pc.search(null, null, null, false)) { System.out.println("Check Package "+t.getId()); List vl = pc.getVersions(t.getId()); diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/comparison/tests/ComparisonTests.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/comparison/tests/ComparisonTests.java index db786420a..e90ef3c57 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/comparison/tests/ComparisonTests.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/comparison/tests/ComparisonTests.java @@ -1,12 +1,9 @@ package org.hl7.fhir.comparison.tests; import com.google.common.base.Charsets; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.NotImplementedException; import org.hl7.fhir.convertors.VersionConvertor_10_50; import org.hl7.fhir.convertors.VersionConvertor_14_50; import org.hl7.fhir.convertors.VersionConvertor_30_50; @@ -14,56 +11,34 @@ import org.hl7.fhir.convertors.VersionConvertor_40_50; import org.hl7.fhir.exceptions.DefinitionException; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.exceptions.FHIRFormatError; -import org.hl7.fhir.exceptions.PathEngineException; import org.hl7.fhir.r5.comparison.CodeSystemComparer; import org.hl7.fhir.r5.comparison.CodeSystemComparer.CodeSystemComparison; import org.hl7.fhir.r5.comparison.ComparisonSession; import org.hl7.fhir.r5.comparison.ValueSetComparer; import org.hl7.fhir.r5.comparison.ValueSetComparer.ValueSetComparison; -import org.hl7.fhir.r5.conformance.ProfileUtilities; import org.hl7.fhir.r5.context.IWorkerContext; -import org.hl7.fhir.r5.context.SimpleWorkerContext; -import org.hl7.fhir.r5.elementmodel.Element; -import org.hl7.fhir.r5.elementmodel.Manager; -import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; -import org.hl7.fhir.r5.elementmodel.ObjectConverter; import org.hl7.fhir.r5.formats.IParser.OutputStyle; import org.hl7.fhir.r5.formats.JsonParser; import org.hl7.fhir.r5.formats.XmlParser; -import org.hl7.fhir.r5.model.Base; import org.hl7.fhir.r5.model.CanonicalResource; import org.hl7.fhir.r5.model.CodeSystem; import org.hl7.fhir.r5.model.Constants; -import org.hl7.fhir.r5.model.FhirPublication; -import org.hl7.fhir.r5.model.Patient; import org.hl7.fhir.r5.model.Resource; -import org.hl7.fhir.r5.model.StructureDefinition; -import org.hl7.fhir.r5.model.TypeDetails; import org.hl7.fhir.r5.model.ValueSet; import org.hl7.fhir.r5.test.utils.TestingUtilities; -import org.hl7.fhir.r5.utils.FHIRPathEngine; -import org.hl7.fhir.r5.utils.FHIRPathEngine.IEvaluationContext; -import org.hl7.fhir.r5.utils.FHIRPathEngine.IEvaluationContext.FunctionDetails; -import org.hl7.fhir.r5.utils.IResourceValidator; -import org.hl7.fhir.r5.utils.IResourceValidator.IValidatorResourceFetcher; -import org.hl7.fhir.r5.utils.IResourceValidator.ReferenceValidationPolicy; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.VersionUtilities; import org.hl7.fhir.utilities.cache.NpmPackage; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; import org.hl7.fhir.utilities.xhtml.XhtmlComposer; -import org.hl7.fhir.validation.ValidationEngine; -import org.hl7.fhir.validation.instance.InstanceValidator; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.thymeleaf.util.IWritableCharSequence; import java.io.File; import java.io.FileNotFoundException; @@ -74,7 +49,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.stream.Stream; @@ -128,7 +102,7 @@ public class ComparisonTests { if (!new File(Utilities.path("[tmp]", "comparison")).exists()) { System.out.println("---- Set up Output ----------------------------------------------------------"); Utilities.createDirectory(Utilities.path("[tmp]", "comparison")); - PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); NpmPackage npm = pcm.loadPackage("hl7.fhir.pubpack", "0.0.4"); for (String f : npm.list("other")) { TextFile.streamToFile(npm.load("other", f), Utilities.path("[tmp]", "comparison", f)); diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/R3R4ConversionTests.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/R3R4ConversionTests.java index ffc44ae11..4a55ec3cc 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/R3R4ConversionTests.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/R3R4ConversionTests.java @@ -21,7 +21,7 @@ import org.hl7.fhir.r4.utils.StructureMapUtilities.ITransformerServices; import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; @@ -42,7 +42,7 @@ import java.util.zip.ZipInputStream; public class R3R4ConversionTests implements ITransformerServices, IValidatorResourceFetcher { private static final boolean SAVING = true; - private PackageCacheManager pcm = null; + private FilesystemPackageCacheManager pcm = null; public static Stream data() throws ParserConfigurationException, SAXException, IOException { if (!(new File(Utilities.path(TestingUtilities.home(), "implementations", "r3maps", "outcomes.json")).exists())) @@ -285,7 +285,7 @@ public class R3R4ConversionTests implements ITransformerServices, IValidatorReso if (contextR3 != null) return; - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); R3ToR4Loader ldr = new R3ToR4Loader().setPatchUrls(true).setKillPrimitives(true); System.out.println("loading R3"); diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/TestingUtilitiesX.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/TestingUtilitiesX.java index 015699f3a..3ef1eb7f3 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/TestingUtilitiesX.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/TestingUtilitiesX.java @@ -1,33 +1,33 @@ package org.hl7.fhir.conversion.tests; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ import java.io.File; @@ -58,7 +58,7 @@ import org.hl7.fhir.utilities.CSFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.VersionUtilities; -import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.ToolsVersion; import org.w3c.dom.Document; @@ -83,9 +83,9 @@ public class TestingUtilitiesX { fcontexts = new HashMap<>(); } if (!fcontexts.containsKey(version)) { - PackageCacheManager pcm; + FilesystemPackageCacheManager pcm; try { - pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); IWorkerContext fcontext = SimpleWorkerContext.fromPackage(pcm.loadPackage(VersionUtilities.packageForVersion(version), version), loaderForVersion(version)); fcontext.setUcumService(new UcumEssenceService(TestingUtilitiesX.loadTestResourceStream("ucum", "ucum-essence.xml"))); fcontext.setExpansionProfile(new Parameters()); diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ProfileComparisonTests.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ProfileComparisonTests.java index 430ebc999..b52f3fb8d 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ProfileComparisonTests.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ProfileComparisonTests.java @@ -1,18 +1,5 @@ package org.hl7.fhir.validation.tests; -import java.io.File; -import java.util.UUID; - -import org.hl7.fhir.convertors.R5ToR5Loader; -import org.hl7.fhir.r5.conformance.ProfileComparer; -import org.hl7.fhir.r5.model.FhirPublication; -import org.hl7.fhir.r5.model.StructureDefinition; -import org.hl7.fhir.r5.utils.KeyGenerator; -import org.hl7.fhir.utilities.cache.PackageCacheManager; -import org.hl7.fhir.utilities.cache.ToolsVersion; -import org.hl7.fhir.r5.test.utils.TestingUtilities; -import org.hl7.fhir.validation.ValidationEngine; -import org.hl7.fhir.validation.tests.utilities.TestUtilities; import org.junit.jupiter.api.Test; public class ProfileComparisonTests { From 37f6a6ed8c0fd534234a67d7d835c223fefb5af9 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Sun, 24 May 2020 18:26:03 -0400 Subject: [PATCH 02/13] Package manager --- .../cache/BasePackageCacheManager.java | 54 ++++++++++++++- .../cache/FilesystemPackageCacheManager.java | 65 ++++++------------- .../utilities/cache/IPackageCacheManager.java | 5 +- 3 files changed, 75 insertions(+), 49 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java index bc7b25ec7..61b19a61a 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java @@ -11,6 +11,13 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { private List myPackageServers = new ArrayList<>(); + /** + * Constructor + */ + public BasePackageCacheManager() { + super(); + } + public List getPackageServers() { return myPackageServers; } @@ -26,12 +33,55 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { } + /** + * Load the latest version of the identified package from the cache - it it exists + * + * @param id + * @return + * @throws IOException + */ + public NpmPackage loadPackageFromCacheOnly(String id) throws IOException { + return loadPackageFromCacheOnly(id, null); + } + + protected abstract NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException; + @Override - public String getPackageId(String canonical) throws IOException { + public String getPackageUrl(String packageId) throws IOException { + String result = null; + NpmPackage npm = loadPackageFromCacheOnly(packageId); + if (npm != null) { + return npm.canonical(); + } + + for (String nextPackageServer : getPackageServers()) { + result = getPackageUrl(packageId, nextPackageServer); + if (result != null) { + return result; + } + } + + return result; + } + + + private String getPackageUrl(String packageId, String server) throws IOException { + PackageClient pc = new PackageClient(server); + List res = pc.search(packageId, null, null, false); + if (res.size() == 0) { + return null; + } else { + return res.get(0).getUrl(); + } + } + + + @Override + public String getPackageId(String canonicalUrl) throws IOException { String result = null; for (String nextPackageServer : getPackageServers()) { - result = getPackageId(canonical, nextPackageServer); + result = getPackageId(canonicalUrl, nextPackageServer); if (result != null) { break; } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java index 7ac21b3c3..f01110b86 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java @@ -80,10 +80,12 @@ import java.util.Map.Entry; * @author Grahame Grieve */ public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager { - private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class); public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$"; public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$"; + private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class); private static final String CACHE_VERSION = "3"; // second version - see wiki page + public static final String PRIMARY_SERVER = "http://packages.fhir.org"; + public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages"; private String cacheFolder; private boolean progress = true; private List temporaryPackages = new ArrayList(); @@ -95,8 +97,8 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple * Constructor */ public FilesystemPackageCacheManager(boolean userMode, int toolsVersion) throws IOException { - addPackageServer("http://packages.fhir.org"); - addPackageServer("http://packages2.fhir.org/packages"); + addPackageServer(PRIMARY_SERVER); + addPackageServer(SECONDARY_SERVER); if (userMode) cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages"); @@ -196,15 +198,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } } - private String getPackageUrl(String packageId, String server) throws IOException { - PackageClient pc = new PackageClient(server); - List res = pc.search(packageId, null, null, false); - if (res.size() == 0) { - return null; - } else { - return res.get(0).getUrl(); - } - } private void listSpecs(Map specList, String server) throws IOException { PackageClient pc = new PackageClient(server); @@ -234,7 +227,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } // ok, well, we'll try the old way - return fetchTheOldWay(id, version); + return fetchTheOldWay(id, version); } public String getLatestVersion(String id) throws IOException { @@ -293,17 +286,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } } - /** - * Load the latest version of the identified package from the cache - it it exists - * - * @param id - * @return - * @throws IOException - */ - public NpmPackage loadPackageFromCacheOnly(String id) throws IOException { - return loadPackageFromCacheOnly(id, null); - } - /** * Load the identified package from the cache - it it exists *

@@ -315,7 +297,8 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple * @return * @throws IOException */ - public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException { + @Override + protected NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException { if (!Utilities.noString(version) && version.startsWith("file:")) { return loadPackageFromFile(id, version.substring(5)); } @@ -345,14 +328,15 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple /** * Add an already fetched package to the cache */ - public NpmPackage addPackageToCache(String id, String version, InputStream tgz, String sourceDesc) throws IOException { + @Override + public NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException { checkValidVersionString(version, id); if (progress) { System.out.println("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache"); System.out.print(" Fetching:"); } - NpmPackage npm = NpmPackage.fromPackage(tgz, sourceDesc, true); + NpmPackage npm = NpmPackage.fromPackage(packageTgzInputStream, sourceDesc, true); if (progress) { System.out.println(); @@ -435,23 +419,11 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple @Override public String getPackageUrl(String packageId) throws IOException { - String result = null; - NpmPackage npm = loadPackageFromCacheOnly(packageId); - if (npm != null) { - return npm.canonical(); - } - - for (String nextPackageServer : getPackageServers()) { - result = getPackageUrl(nextPackageServer); - if (result != null) { - return result; - } - } - + String result = super.getPackageUrl(packageId); if (result == null) { result = getPackageUrlFromBuildList(packageId); } - + return result; } @@ -459,8 +431,9 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple for (NpmPackage p : temporaryPackages) { specList.put(p.name(), p.canonical()); } - listSpecs(specList, PRIMARY_SERVER); - listSpecs(specList, SECONDARY_SERVER); + for (String next : getPackageServers()) { + listSpecs(specList, next); + } addCIBuildSpecs(specList); } @@ -550,11 +523,11 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } @Override - public String getPackageId(String canonical) throws IOException { - String retVal = super.getPackageId(canonical); + public String getPackageId(String canonicalUrl) throws IOException { + String retVal = super.getPackageId(canonicalUrl); if (retVal == null) { - retVal = getPackageIdFromBuildList(canonical); + retVal = getPackageIdFromBuildList(canonicalUrl); } return retVal; diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java index 40a6e61c7..5ff0529c4 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java @@ -3,11 +3,14 @@ package org.hl7.fhir.utilities.cache; import org.hl7.fhir.exceptions.FHIRException; import java.io.IOException; +import java.io.InputStream; public interface IPackageCacheManager { - String getPackageId(String canonical) throws IOException; + String getPackageId(String canonicalUrl) throws IOException; + + NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException; String getPackageUrl(String packageId) throws IOException; From c0f6366626ef1379e4583cee7e6dce6e16ac0825 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Wed, 27 May 2020 05:27:44 -0400 Subject: [PATCH 03/13] NPM rework --- .../cache/BasePackageCacheManager.java | 60 +++++++++++++--- ...kageClient.java => BasePackageClient.java} | 42 +++++------ .../utilities/cache/CachingPackageClient.java | 71 +++++++++++++++++++ .../cache/FilesystemPackageCacheManager.java | 48 ++++--------- .../utilities/cache/IPackageCacheManager.java | 1 - .../hl7/fhir/utilities/cache/NpmPackage.java | 21 +++--- ...ts.java => CachingPackageClientTests.java} | 28 ++++---- .../validation/packages/PackageValidator.java | 6 +- 8 files changed, 179 insertions(+), 98 deletions(-) rename org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/{PackageClient.java => BasePackageClient.java} (92%) create mode 100644 org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java rename org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/{PackageClientTests.java => CachingPackageClientTests.java} (70%) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java index 61b19a61a..850590ed7 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java @@ -1,14 +1,19 @@ package org.hl7.fhir.utilities.cache; import org.apache.commons.lang3.Validate; +import org.hl7.fhir.utilities.Utilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; public abstract class BasePackageCacheManager implements IPackageCacheManager { - + private static final Logger ourLog = LoggerFactory.getLogger(BasePackageCacheManager.class); private List myPackageServers = new ArrayList<>(); /** @@ -35,16 +40,37 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { /** * Load the latest version of the identified package from the cache - it it exists - * - * @param id - * @return - * @throws IOException */ public NpmPackage loadPackageFromCacheOnly(String id) throws IOException { return loadPackageFromCacheOnly(id, null); } - protected abstract NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException; + /** + * Try to load a package from all registered package servers, and return null + * if it can not be found at any of them. + */ + @Nullable + protected InputStreamWithSrc loadFromPackageServer(String id, String version) { + + for (String nextPackageServer : getPackageServers()) { + CachingPackageClient packageClient = new CachingPackageClient(nextPackageServer); + try { + if (Utilities.noString(version)) { + version = packageClient.getLatestVersion(id); + } + InputStream stream = packageClient.fetch(id, version); + String url = packageClient.url(id, version); + return new InputStreamWithSrc(stream, packageClient.url(id, version), version); + } catch (IOException e) { + ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer); + } + + } + + return null; + } + + protected abstract NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException; @Override public String getPackageUrl(String packageId) throws IOException { @@ -66,8 +92,8 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { private String getPackageUrl(String packageId, String server) throws IOException { - PackageClient pc = new PackageClient(server); - List res = pc.search(packageId, null, null, false); + CachingPackageClient pc = new CachingPackageClient(server); + List res = pc.search(packageId, null, null, false); if (res.size() == 0) { return null; } else { @@ -91,13 +117,13 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { } private String getPackageId(String canonical, String server) throws IOException { - PackageClient pc = new PackageClient(server); - List res = pc.search(null, canonical, null, false); + CachingPackageClient pc = new CachingPackageClient(server); + List res = pc.search(null, canonical, null, false); if (res.size() == 0) { return null; } else { // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others) - for (PackageClient.PackageInfo pi : res) { + for (CachingPackageClient.PackageInfo pi : res) { if (canonical.equals(pi.getCanonical())) { return pi.getId(); } @@ -106,5 +132,17 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { } } + public class InputStreamWithSrc { + + public InputStream stream; + public String url; + public String version; + + public InputStreamWithSrc(InputStream stream, String url, String version) { + this.stream = stream; + this.url = url; + this.version = version; + } + } } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java similarity index 92% rename from org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java rename to org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java index 16f6bb5e0..da85dc16d 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java @@ -1,5 +1,14 @@ package org.hl7.fhir.utilities.cache; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.VersionUtilities; +import org.hl7.fhir.utilities.json.JSONUtil; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -13,19 +22,7 @@ import java.util.Date; import java.util.List; import java.util.Set; -import org.hl7.fhir.exceptions.FHIRException; -import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; -import org.hl7.fhir.utilities.TextFile; -import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.VersionUtilities; -import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; -import org.hl7.fhir.utilities.json.JSONUtil; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -public class PackageClient { +public abstract class BasePackageClient { public class PackageInfo { private String id; @@ -34,7 +31,7 @@ public class PackageClient { private String description; private String url; private String canonical; - + public PackageInfo(String id, String version, String fhirVersion, String description, String url, String canonical) { super(); this.id = id; @@ -62,21 +59,21 @@ public class PackageClient { public String getUrl() { return url; } - + public String getCanonical() { return canonical; } @Override public String toString() { return id+"#"+(version == null ? "?pc-pi?" : version)+(fhirVersion == null ? "": " ("+canonical+") for FHIR "+fhirVersion)+(url == null ? "" : " @"+url)+(description == null ? "" : " '"+description+"'"); - } + } } - + private String address; private String cacheFolder; - public PackageClient(String address) { + public BasePackageClient(String address) { super(); this.address = address; try { @@ -109,15 +106,10 @@ public class PackageClient { } public InputStream fetchCached(String url) throws IOException, FileNotFoundException { - File cacheFile = new File(Utilities.path(cacheFolder, fn(url))); - if (cacheFile.exists()) { - return new FileInputStream(cacheFile); - } - TextFile.bytesToFile(TextFile.streamToBytes(fetchUrl(url, null)), cacheFile); - return new FileInputStream(cacheFile); + return fetchUrl(url, null); } - private String fn(String url) { + protected String fn(String url) { String[] p = url.split("\\/"); return p[2]+"-"+p[p.length-2]+"-"+p[p.length-1]+".tgz"; } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java new file mode 100644 index 000000000..cdb1d66d6 --- /dev/null +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java @@ -0,0 +1,71 @@ +package org.hl7.fhir.utilities.cache; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Set; + +import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.VersionUtilities; +import org.hl7.fhir.utilities.json.JSONUtil; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +/** + * Implementation of a package client that keeps a local disk cache of downloaded artifacts + * in order to avoid re-downloading things + */ +public class CachingPackageClient extends BasePackageClient { + + private String cacheFolder; + + + public CachingPackageClient(String address) { + super(address); + try { + cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "package-client"); + Utilities.createDirectory(cacheFolder); + } catch (IOException e) { + } + } + + public boolean exists(String id, String ver) throws IOException { + List vl = getVersions(id); + for (PackageInfo pi : vl) { + if (ver.equals(pi.getVersion())) { + return true; + } + } + return false; + } + + public InputStream fetchCached(String url) throws IOException { + File cacheFile = new File(Utilities.path(cacheFolder, fn(url))); + if (cacheFile.exists()) { + return new FileInputStream(cacheFile); + } + InputStream fetchedPackage = super.fetchCached(url); + TextFile.bytesToFile(TextFile.streamToBytes(fetchedPackage), cacheFile); + return new FileInputStream(cacheFile); + } + + + public Date getNewPackages(Date lastCalled, List updates) { + return null; + } + + + +} \ No newline at end of file diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java index f01110b86..3d0e528b9 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java @@ -40,7 +40,6 @@ import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder; -import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; import org.hl7.fhir.utilities.json.JSONUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +66,8 @@ import java.util.Map; import java.util.Map.Entry; /** - * Package cache manager + * This is a package cache manager implementation that uses a local disk cache + * *

* API: *

@@ -200,30 +200,19 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple private void listSpecs(Map specList, String server) throws IOException { - PackageClient pc = new PackageClient(server); - List matches = pc.search(null, null, null, false); - for (PackageInfo m : matches) { + CachingPackageClient pc = new CachingPackageClient(server); + List matches = pc.search(null, null, null, false); + for (BasePackageClient.PackageInfo m : matches) { if (!specList.containsKey(m.getId())) { specList.put(m.getId(), m.getUrl()); } } } - private InputStreamWithSrc loadFromPackageServer(String id, String version) { - - for (String nextPackageServer : getPackageServers()) { - PackageClient packageClient = new PackageClient(nextPackageServer); - try { - if (Utilities.noString(version)) { - version = packageClient.getLatestVersion(id); - } - InputStream stream = packageClient.fetch(id, version); - String url = packageClient.url(id, version); - return new InputStreamWithSrc(stream, packageClient.url(id, version), version); - } catch (IOException e) { - ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer); - } - + protected InputStreamWithSrc loadFromPackageServer(String id, String version) { + InputStreamWithSrc retVal = super.loadFromPackageServer(id, version); + if (retVal != null) { + return retVal; } // ok, well, we'll try the old way @@ -232,7 +221,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple public String getLatestVersion(String id) throws IOException { for (String nextPackageServer : getPackageServers()) { - PackageClient pc = new PackageClient(nextPackageServer); + CachingPackageClient pc = new CachingPackageClient(nextPackageServer); try { return pc.getLatestVersion(id); } catch (IOException e) { @@ -443,6 +432,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple if (!Utilities.noString(version) && version.startsWith("file:")) { return loadPackageFromFile(id, version.substring(5)); } + NpmPackage p = loadPackageFromCacheOnly(id, version); if (p != null) { if ("current".equals(version)) { @@ -461,7 +451,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple } // nup, don't have it locally (or it's expired) - InputStreamWithSrc source; + FilesystemPackageCacheManager.InputStreamWithSrc source; if ("current".equals(version)) { // special case - fetch from ci-build server source = loadFromCIBuild(id); @@ -471,6 +461,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url); } + private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException { try { URL url = new URL(source); @@ -732,19 +723,6 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple InputStream resolvePackage(String packageId, String version); } - public class InputStreamWithSrc { - - public InputStream stream; - public String url; - public String version; - - public InputStreamWithSrc(InputStream stream, String url, String version) { - this.stream = stream; - this.url = url; - this.version = version; - } - } - public class BuildRecordSorter implements Comparator { @Override diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java index 5ff0529c4..8dd49d98e 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/IPackageCacheManager.java @@ -7,7 +7,6 @@ import java.io.InputStream; public interface IPackageCacheManager { - String getPackageId(String canonicalUrl) throws IOException; NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException; diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java index e2b047ba1..d022e661a 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java @@ -102,7 +102,7 @@ public class NpmPackage { public class NpmPackageFolder { private String name; private Map> types = new HashMap<>(); - private Map content = new HashMap(); + private Map content = new HashMap<>(); private JsonObject index; private File folder; @@ -111,6 +111,10 @@ public class NpmPackage { this.name = name; } + public Map> getTypes() { + return types; + } + public String getName() { return name; } @@ -192,15 +196,12 @@ public class NpmPackage { private Map folders = new HashMap<>(); private boolean changedByLoader; // internal qa only! + /** + * Constructor + */ private NpmPackage() { - + super(); } - // public NpmPackage(JsonObject npm, Map content, List folders) { - // this.path = null; - // this.content = content; - // this.npm = npm; - // this.folders = folders; - // } public static NpmPackage fromFolder(String path) throws IOException { NpmPackage res = new NpmPackage(); @@ -839,7 +840,9 @@ public class NpmPackage { stream.write(b); } - + /** + * Keys are resource type names, values are filenames + */ public Map> getTypes() { return folders.get("package").types; } diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageClientTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java similarity index 70% rename from org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageClientTests.java rename to org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java index 345829786..126bfb6bc 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/PackageClientTests.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java @@ -1,18 +1,18 @@ package org.hl7.fhir.utilities.tests; -import org.hl7.fhir.utilities.cache.PackageClient; -import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; +import org.hl7.fhir.utilities.cache.CachingPackageClient; +import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.List; -public class PackageClientTests { +public class CachingPackageClientTests { @Test public void testExists() throws IOException { - PackageClient client = new PackageClient("http://packages.fhir.org"); + CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org"); Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1")); Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2")); Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1")); @@ -20,7 +20,7 @@ public class PackageClientTests { @Test public void testSearch() throws IOException { - PackageClient client = new PackageClient("http://packages.fhir.org"); + CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org"); List matches = client.search("core", null, null, false); for (PackageInfo pi : matches) { System.out.println(pi.toString()); @@ -30,14 +30,14 @@ public class PackageClientTests { @Test public void testSearchNoMatches() throws IOException { - PackageClient client = new PackageClient("http://packages.fhir.org"); + CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org"); List matches = client.search("corezxxx", null, null, false); Assertions.assertTrue(matches.size() == 0); } @Test public void testVersions() throws IOException { - PackageClient client = new PackageClient("http://packages.fhir.org"); + CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org"); List matches = client.getVersions("Simplifier.Core.STU3"); for (PackageInfo pi : matches) { System.out.println(pi.toString()); @@ -47,14 +47,14 @@ public class PackageClientTests { @Test public void testVersionsNone() throws IOException { - PackageClient client = new PackageClient("http://packages.fhir.org"); + CachingPackageClient client = new CachingPackageClient("http://packages.fhir.org"); List matches = client.getVersions("Simplifier.Core.STU3X"); Assertions.assertTrue(matches.size() == 0); } @Test public void testExists2() throws IOException { - PackageClient client = new PackageClient("http://packages2.fhir.org/packages"); + CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages"); Assertions.assertTrue(client.exists("hl7.fhir.r4.core", "4.0.1")); Assertions.assertTrue(!client.exists("hl7.fhir.r4.core", "1.0.2")); Assertions.assertTrue(!client.exists("hl7.fhir.nothing", "1.0.1")); @@ -62,7 +62,7 @@ public class PackageClientTests { @Test public void testSearch2() throws IOException { - PackageClient client = new PackageClient("http://packages2.fhir.org/packages"); + CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages"); List matches = client.search("core", null, null, false); for (PackageInfo pi : matches) { System.out.println(pi.toString()); @@ -72,14 +72,14 @@ public class PackageClientTests { @Test public void testSearchNoMatches2() throws IOException { - PackageClient client = new PackageClient("http://packages2.fhir.org/packages"); + CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages"); List matches = client.search("corezxxx", null, null, false); Assertions.assertTrue(matches.size() == 0); } @Test public void testVersions2() throws IOException { - PackageClient client = new PackageClient("http://packages2.fhir.org/packages"); + CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages"); List matches = client.getVersions("Simplifier.Core.STU3"); for (PackageInfo pi : matches) { System.out.println(pi.toString()); @@ -89,7 +89,7 @@ public class PackageClientTests { @Test public void testVersions2A() throws IOException { - PackageClient client = new PackageClient("http://packages2.fhir.org/packages"); + CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages"); List matches = client.getVersions("hl7.fhir.us.core"); for (PackageInfo pi : matches) { System.out.println(pi.toString()); @@ -99,7 +99,7 @@ public class PackageClientTests { @Test public void testVersionsNone2() throws IOException { - PackageClient client = new PackageClient("http://packages2.fhir.org/packages"); + CachingPackageClient client = new CachingPackageClient("http://packages2.fhir.org/packages"); List matches = client.getVersions("Simplifier.Core.STU3X"); Assertions.assertTrue(matches.size() == 0); } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java index 3837bcf7f..3f9e60412 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java @@ -9,8 +9,8 @@ import org.hl7.fhir.exceptions.FHIRFormatError; import org.hl7.fhir.utilities.VersionUtilities; import org.hl7.fhir.utilities.cache.NpmPackage; import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; -import org.hl7.fhir.utilities.cache.PackageClient; -import org.hl7.fhir.utilities.cache.PackageClient.PackageInfo; +import org.hl7.fhir.utilities.cache.CachingPackageClient; +import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo; import org.hl7.fhir.utilities.cache.ToolsVersion; public class PackageValidator { @@ -22,7 +22,7 @@ public class PackageValidator { private void execute() throws IOException { FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION); - PackageClient pc = new PackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER); + CachingPackageClient pc = new CachingPackageClient(FilesystemPackageCacheManager.PRIMARY_SERVER); for (PackageInfo t : pc.search(null, null, null, false)) { System.out.println("Check Package "+t.getId()); List vl = pc.getVersions(t.getId()); From 91e2d227fb24418ea3046b3aafd10cf15bffbfdf Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Thu, 28 May 2020 21:07:18 -0400 Subject: [PATCH 04/13] Compile fixes --- .../org/hl7/fhir/utilities/cache/BasePackageCacheManager.java | 2 +- .../hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java | 2 +- .../org/hl7/fhir/utilities/tests/CachingPackageClientTests.java | 2 +- .../java/org/hl7/fhir/validation/packages/PackageValidator.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java index 850590ed7..105f11d9a 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java @@ -70,7 +70,7 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { return null; } - protected abstract NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException; + public abstract NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException; @Override public String getPackageUrl(String packageId) throws IOException { diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java index 3d0e528b9..9ea839180 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java @@ -287,7 +287,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple * @throws IOException */ @Override - protected NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException { + public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException { if (!Utilities.noString(version) && version.startsWith("file:")) { return loadPackageFromFile(id, version.substring(5)); } diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java index 126bfb6bc..21485daff 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/tests/CachingPackageClientTests.java @@ -1,7 +1,7 @@ package org.hl7.fhir.utilities.tests; import org.hl7.fhir.utilities.cache.CachingPackageClient; -import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo; +import org.hl7.fhir.utilities.cache.BasePackageClient.PackageInfo; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java index 3f9e60412..c6d9890be 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/packages/PackageValidator.java @@ -10,7 +10,7 @@ import org.hl7.fhir.utilities.VersionUtilities; import org.hl7.fhir.utilities.cache.NpmPackage; import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.cache.CachingPackageClient; -import org.hl7.fhir.utilities.cache.CachingPackageClient.PackageInfo; +import org.hl7.fhir.utilities.cache.BasePackageClient.PackageInfo; import org.hl7.fhir.utilities.cache.ToolsVersion; public class PackageValidator { From 9fe0688ee51eeae607beb8392a7b77da4e0e505e Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Sat, 30 May 2020 11:08:33 -0400 Subject: [PATCH 05/13] Work on rework --- .../utilities/cache/CachingPackageClient.java | 15 +-------------- .../cache/FilesystemPackageCacheManager.java | 4 ++-- ...{BasePackageClient.java => PackageClient.java} | 4 ++-- 3 files changed, 5 insertions(+), 18 deletions(-) rename org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/{BasePackageClient.java => PackageClient.java} (98%) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java index cdb1d66d6..571dd59c9 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/CachingPackageClient.java @@ -2,32 +2,19 @@ package org.hl7.fhir.utilities.cache; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.Set; -import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.VersionUtilities; -import org.hl7.fhir.utilities.json.JSONUtil; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; /** * Implementation of a package client that keeps a local disk cache of downloaded artifacts * in order to avoid re-downloading things */ -public class CachingPackageClient extends BasePackageClient { +public class CachingPackageClient extends PackageClient { private String cacheFolder; diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java index 3d0e528b9..fff778f59 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/FilesystemPackageCacheManager.java @@ -201,8 +201,8 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple private void listSpecs(Map specList, String server) throws IOException { CachingPackageClient pc = new CachingPackageClient(server); - List matches = pc.search(null, null, null, false); - for (BasePackageClient.PackageInfo m : matches) { + List matches = pc.search(null, null, null, false); + for (PackageClient.PackageInfo m : matches) { if (!specList.containsKey(m.getId())) { specList.put(m.getId(), m.getUrl()); } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java similarity index 98% rename from org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java rename to org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java index da85dc16d..f384d2f2e 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageClient.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageClient.java @@ -22,7 +22,7 @@ import java.util.Date; import java.util.List; import java.util.Set; -public abstract class BasePackageClient { +public class PackageClient { public class PackageInfo { private String id; @@ -73,7 +73,7 @@ public abstract class BasePackageClient { private String cacheFolder; - public BasePackageClient(String address) { + public PackageClient(String address) { super(); this.address = address; try { From 0d8cab0b6e146e7f0c359c0fc6f5d6399c3edc3b Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Tue, 2 Jun 2020 07:45:54 -0400 Subject: [PATCH 06/13] A bit more rework --- .../hl7/fhir/utilities/cache/BasePackageCacheManager.java | 2 +- .../main/java/org/hl7/fhir/utilities/cache/NpmPackage.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java index 105f11d9a..a6b475243 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java @@ -60,7 +60,7 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { } InputStream stream = packageClient.fetch(id, version); String url = packageClient.url(id, version); - return new InputStreamWithSrc(stream, packageClient.url(id, version), version); + return new InputStreamWithSrc(stream, url, version); } catch (IOException e) { ourLog.info("Failed to resolve package {}#{} from server: {}", id, version, nextPackageServer); } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java index a49d83494..200b86c41 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java @@ -195,6 +195,7 @@ public class NpmPackage { private JsonObject npm; private Map folders = new HashMap<>(); private boolean changedByLoader; // internal qa only! + private Map userData = new HashMap<>(); /** * Constructor @@ -210,6 +211,10 @@ public class NpmPackage { return res; } + public Map getUserData() { + return userData; + } + public static void loadFiles(NpmPackage res, String path, File source, String... exemptions) throws FileNotFoundException, IOException { res.npm = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(Utilities.path(path, "package", "package.json"))); res.path = path; From 56d933199cc38124d3571b382d98194383356914 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Tue, 2 Jun 2020 18:02:25 -0400 Subject: [PATCH 07/13] A bit of package rework --- .../hl7/fhir/utilities/cache/NpmPackage.java | 20 +++++- .../utilities/cache/PackageGenerator.java | 67 +++++++++++-------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java index 200b86c41..aa8dd1419 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/NpmPackage.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -204,6 +205,9 @@ public class NpmPackage { super(); } + /** + * Factory method that parses a package from an extracted folder + */ public static NpmPackage fromFolder(String path) throws IOException { NpmPackage res = new NpmPackage(); loadFiles(res, path, new File(path)); @@ -211,6 +215,15 @@ public class NpmPackage { return res; } + /** + * Factory method that starts a new empty package using the given PackageGenerator to create the manifest + */ + public static NpmPackage empty(PackageGenerator thePackageGenerator) { + NpmPackage retVal = new NpmPackage(); + retVal.npm = thePackageGenerator.getRootJsonObject(); + return retVal; + } + public Map getUserData() { return userData; } @@ -787,7 +800,7 @@ public class NpmPackage { TextFile.bytesToFile(b, Utilities.path(dir.getAbsolutePath(), n, s)); } } - byte[] cnt = indexer.build().getBytes(Charset.forName("UTF-8")); + byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8); TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), n, ".index.json")); } byte[] cnt = TextFile.stringToBytes(new GsonBuilder().setPrettyPrinting().create().toJson(npm), false); @@ -825,7 +838,7 @@ public class NpmPackage { tar.closeArchiveEntry(); } } - byte[] cnt = indexer.build().getBytes(Charset.forName("UTF-8")); + byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8); TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json"); entry.setSize(cnt.length); tar.putArchiveEntry(entry); @@ -942,6 +955,9 @@ public class NpmPackage { } public void addFile(String folderName, String name, byte[] cnt, String type) { + if (!folders.containsKey(folderName)) { + folders.put(folderName, new NpmPackageFolder(folderName)); + } NpmPackageFolder folder = folders.get(folderName); folder.content.put(name, cnt); if (!folder.types.containsKey(type)) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java index 03724b647..9f59dd234 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java @@ -1,33 +1,33 @@ package org.hl7.fhir.utilities.cache; -/* - Copyright (c) 2011+, HL7, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of HL7 nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - */ +/* + Copyright (c) 2011+, HL7, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of HL7 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + */ @@ -63,9 +63,14 @@ public class PackageGenerator { throw new Error("Unknown Type"); } } + private OutputStream stream; private JsonObject object; + public PackageGenerator() { + object = new JsonObject(); + } + public PackageGenerator(OutputStream stream) { super(); this.stream = stream; @@ -79,7 +84,11 @@ public class PackageGenerator { object = parser.parse(TextFile.streamToString(template)).getAsJsonObject(); } - + + public JsonObject getRootJsonObject() { + return object; + } + public void commit() throws IOException { Gson gson = new GsonBuilder().setPrettyPrinting().create(); String json = gson.toJson(object); From 20fab5e047d003f9e5c45f59bcd92ffe762f2382 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Tue, 2 Jun 2020 18:16:08 -0400 Subject: [PATCH 08/13] Rework package generator --- .../cache/BasePackageCacheManager.java | 22 ++++++++++++++----- .../utilities/cache/PackageGenerator.java | 3 ++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java index a6b475243..07bdff1a7 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/BasePackageCacheManager.java @@ -11,10 +11,12 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; public abstract class BasePackageCacheManager implements IPackageCacheManager { private static final Logger ourLog = LoggerFactory.getLogger(BasePackageCacheManager.class); private List myPackageServers = new ArrayList<>(); + private Function myClientFactory = address -> new CachingPackageClient(address); /** * Constructor @@ -23,6 +25,14 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { super(); } + /** + * Provide a new client factory implementation + */ + public void setClientFactory(Function theClientFactory) { + Validate.notNull(theClientFactory, "theClientFactory must not be null"); + myClientFactory = theClientFactory; + } + public List getPackageServers() { return myPackageServers; } @@ -53,7 +63,7 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { protected InputStreamWithSrc loadFromPackageServer(String id, String version) { for (String nextPackageServer : getPackageServers()) { - CachingPackageClient packageClient = new CachingPackageClient(nextPackageServer); + PackageClient packageClient = myClientFactory.apply(nextPackageServer); try { if (Utilities.noString(version)) { version = packageClient.getLatestVersion(id); @@ -92,8 +102,8 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { private String getPackageUrl(String packageId, String server) throws IOException { - CachingPackageClient pc = new CachingPackageClient(server); - List res = pc.search(packageId, null, null, false); + PackageClient pc = myClientFactory.apply(server); + List res = pc.search(packageId, null, null, false); if (res.size() == 0) { return null; } else { @@ -117,13 +127,13 @@ public abstract class BasePackageCacheManager implements IPackageCacheManager { } private String getPackageId(String canonical, String server) throws IOException { - CachingPackageClient pc = new CachingPackageClient(server); - List res = pc.search(null, canonical, null, false); + PackageClient pc = myClientFactory.apply(server); + List res = pc.search(null, canonical, null, false); if (res.size() == 0) { return null; } else { // this is driven by HL7 Australia (http://hl7.org.au/fhir/ is the canonical url for the base package, and the root for all the others) - for (CachingPackageClient.PackageInfo pi : res) { + for (PackageClient.PackageInfo pi : res) { if (canonical.equals(pi.getCanonical())) { return pi.getId(); } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java index 9f59dd234..a39e537a7 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageGenerator.java @@ -100,7 +100,8 @@ public class PackageGenerator { } public PackageGenerator name(String value) { - object.addProperty("name", "@fhir/"+value); + // NOTE: I removed a prefix of "@fhir/" here. What was this for? -JA + object.addProperty("name", value); return this; } From cc0bd10835d58f1b510776b43c57d449b761301f Mon Sep 17 00:00:00 2001 From: markiantorno Date: Wed, 3 Jun 2020 19:18:13 -0400 Subject: [PATCH 09/13] I have no idea what I'm doing --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 84be315c6..defa980e6 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,9 @@ 1.8 false UTF-8 - false + + true 512m 4000m true @@ -159,6 +161,9 @@ maven-surefire-plugin ${maven_surefire_version} + 6 + true + classes false false