breaking up package installer and package config (#4319)

* breaking up package installer and package config

* breaking out parsing of resources

* wrong nullable

* added an integration test

* adding a changelog

* added a few minor tests

* review fixes

* unit test fixes

* fixing code checkstyle

* moving fetching local package data to package loader svc

* updating the method name

* splitting the fhirversion parsing

* updating table creation task for pretty printing and use in RTE

* adding foreign keys

* Adding mroe info

* minor prettiness changes

* fixing fk generation

* minor changes

* update version

Co-authored-by: leif stawnyczy <leifstawnyczy@leifs-mbp.home>
Co-authored-by: leif stawnyczy <leifstawnyczy@leifs-MacBook-Pro.local>

.gitignore

@@ -168,4 +168,5 @@ Snap.*
 
 
 /database/
+/activemq-data/
 /.run/

hapi-deployable-pom/pom.xml

@@ -4,7 +4,7 @@
     <parent>
 		 <groupId>ca.uhn.hapi.fhir</groupId>
 		 <artifactId>hapi-fhir</artifactId>
-		 <version>6.3.8-SNAPSHOT</version>
+		 <version>6.3.9-SNAPSHOT</version>
 		 <relativePath>../pom.xml</relativePath>
 	 </parent>
 

hapi-fhir-android/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-base/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-bom/pom.xml

@@ -4,14 +4,14 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>ca.uhn.hapi.fhir</groupId>
 	<artifactId>hapi-fhir-bom</artifactId>
-	<version>6.3.8-SNAPSHOT</version>
+	<version>6.3.9-SNAPSHOT</version>
 	<packaging>pom</packaging>
 	<name>HAPI FHIR BOM</name>
 
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-checkstyle/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-api/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/BaseCommand.java

@@ -579,20 +579,24 @@ public abstract class BaseCommand implements Comparable<BaseCommand> {
 		return value;
 	}
 
-	protected void parseFhirContext(CommandLine theCommandLine) throws ParseException {
+	protected FhirVersionEnum parseFhirVersion(CommandLine theCommandLine) throws ParseException {
 		String version = theCommandLine.getOptionValue(FHIR_VERSION_PARAM);
 		if (isBlank(version)) {
 			throw new ParseException(Msg.code(1581) + "Missing required option: -" + FHIR_VERSION_PARAM);
 		}
-
 		try {
 			FhirVersionEnum versionEnum = FhirVersionEnum.valueOf(version.toUpperCase());
-			myFhirCtx = versionEnum.newContext();
+			return versionEnum;
 		} catch (Exception e) {
 			throw new ParseException(Msg.code(1582) + "Invalid FHIR version string: " + version);
 		}
 	}
 
+	protected void parseFhirContext(CommandLine theCommandLine) throws ParseException {
+		FhirVersionEnum versionEnum = parseFhirVersion(theCommandLine);
+		myFhirCtx = versionEnum.newContext();
+	}
+
 
 	public abstract void run(CommandLine theCommandLine) throws ParseException, ExecutionException;
 

hapi-fhir-cli/hapi-fhir-cli-app/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-cli</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom</relativePath>
 	</parent>
 

hapi-fhir-cli/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-client-okhttp/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-converter/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-dist/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-docs/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4309-breakout-package-loader-and-resource-parser.yaml

@@ -0,0 +1,8 @@
+---
+type: fix
+issue: 4309
+title: "
+      Broke out services for loading npm packages
+      and parsing npm package resources that will
+      not be dependent on any the DAO layer.
+      "

hapi-fhir-jacoco/pom.xml

@@ -11,7 +11,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jaxrsserver-base/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpa/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-jpaserver-base/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java

@@ -63,6 +63,7 @@ import ca.uhn.fhir.jpa.packages.IPackageInstallerSvc;
 import ca.uhn.fhir.jpa.packages.JpaPackageCache;
 import ca.uhn.fhir.jpa.packages.NpmJpaValidationSupport;
 import ca.uhn.fhir.jpa.packages.PackageInstallerSvcImpl;
+import ca.uhn.fhir.jpa.packages.util.PackageUtils;
 import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
 import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
 import ca.uhn.fhir.jpa.partition.PartitionLookupSvcImpl;
@@ -148,7 +149,6 @@ import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
 import ca.uhn.hapi.converters.canonical.VersionCanonicalizer;
 import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport;
 import org.hl7.fhir.utilities.graphql.IGraphQLStorageServices;
-import org.hl7.fhir.utilities.npm.PackageClient;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -195,7 +195,8 @@ import java.util.Date;
 	ValidationSupportConfig.class,
 	Batch2SupportConfig.class,
 	JpaBulkExportConfig.class,
-	SearchConfig.class
+	SearchConfig.class,
+	PackageLoaderConfig.class
 })
 public class JpaConfig {
 	public static final String JPA_VALIDATION_SUPPORT_CHAIN = "myJpaValidationSupportChain";
@@ -305,13 +306,9 @@ public class JpaConfig {
 		return new DaoResourceLinkResolver<JpaPid>();
 	}
 
-	@Bean
+	@Bean(name = PackageUtils.LOADER_WITH_CACHE)
 	public IHapiPackageCacheManager packageCacheManager() {
-		JpaPackageCache retVal = new JpaPackageCache();
+		return new JpaPackageCache();
-		retVal.getPackageServers().clear();
-		retVal.getPackageServers().add(PackageClient.PRIMARY_SERVER);
-		retVal.getPackageServers().add(PackageClient.SECONDARY_SERVER);
-		return retVal;
 	}
 
 	@Bean

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/PackageLoaderConfig.java

@@ -0,0 +1,26 @@
+package ca.uhn.fhir.jpa.config;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.packages.loader.PackageLoaderSvc;
+import ca.uhn.fhir.jpa.packages.loader.PackageResourceParsingSvc;
+import org.hl7.fhir.utilities.npm.PackageClient;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class PackageLoaderConfig {
+
+	@Bean
+	public PackageLoaderSvc packageLoaderSvc() {
+		PackageLoaderSvc svc = new PackageLoaderSvc();
+		svc.getPackageServers().clear();
+		svc.getPackageServers().add(PackageClient.PRIMARY_SERVER);
+		svc.getPackageServers().add(PackageClient.SECONDARY_SERVER);
+		return svc;
+	}
+
+	@Bean
+	public PackageResourceParsingSvc resourceParsingSvc(FhirContext theContext) {
+		return new PackageResourceParsingSvc(theContext);
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java

@@ -41,23 +41,18 @@ import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionResourceEntity;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
+import ca.uhn.fhir.jpa.packages.loader.NpmPackageData;
+import ca.uhn.fhir.jpa.packages.loader.PackageLoaderSvc;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.util.BinaryUtil;
-import ca.uhn.fhir.util.ClasspathUtil;
 import ca.uhn.fhir.util.ResourceUtil;
 import ca.uhn.fhir.util.StringUtil;
 import org.apache.commons.collections4.comparators.ReverseComparator;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.Validate;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.instance.model.api.IBaseBinary;
 import org.hl7.fhir.instance.model.api.IBaseResource;
@@ -89,11 +84,7 @@ import javax.persistence.criteria.Root;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -131,9 +122,43 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 	@Autowired
 	private PartitionSettings myPartitionSettings;
 
+	@Autowired
+	private PackageLoaderSvc myPackageLoaderSvc;
+
 	@Autowired(required = false)//It is possible that some implementers will not create such a bean.
 	private IBinaryStorageSvc myBinaryStorageSvc;
 
+	@Override
+	public void addPackageServer(@Nonnull String theUrl) {
+		assert myPackageLoaderSvc != null;
+		myPackageLoaderSvc.addPackageServer(theUrl);
+	}
+
+	@Override
+	public String getPackageId(String theS) throws IOException {
+		return myPackageLoaderSvc.getPackageId(theS);
+	}
+
+	@Override
+	public void setSilent(boolean silent) {
+		myPackageLoaderSvc.setSilent(silent);
+	}
+
+	@Override
+	public String getPackageUrl(String theS) throws IOException {
+		return myPackageLoaderSvc.getPackageUrl(theS);
+	}
+
+	@Override
+	public List<String> getPackageServers() {
+		return myPackageLoaderSvc.getPackageServers();
+	}
+
+	@Override
+	protected BasePackageCacheManager.InputStreamWithSrc loadFromPackageServer(String id, String version) {
+		throw new UnsupportedOperationException(Msg.code(2220) + "Use PackageLoaderSvc for loading packages.");
+	}
+
 	@Override
 	@Transactional
 	public NpmPackage loadPackageFromCacheOnly(String theId, @Nullable String theVersion) {
@@ -221,22 +246,19 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 		return myDaoRegistry.getResourceDao("Binary");
 	}
 
-	@Override
+	private NpmPackage addPackageToCacheInternal(
-	public NpmPackage addPackageToCache(String thePackageId, String thePackageVersionId, InputStream thePackageTgzInputStream, String theSourceDesc) throws IOException {
+		NpmPackageData thePackageData
-		Validate.notBlank(thePackageId, "thePackageId must not be null");
+	) {
-		Validate.notBlank(thePackageVersionId, "thePackageVersionId must not be null");
+		NpmPackage npmPackage = thePackageData.getPackage();
-		Validate.notNull(thePackageTgzInputStream, "thePackageTgzInputStream must not be null");
+		String packageId = thePackageData.getPackageId();
+		String initialPackageVersionId = thePackageData.getPackageVersionId();
+		byte[] bytes = thePackageData.getBytes();
 
-		byte[] bytes = IOUtils.toByteArray(thePackageTgzInputStream);
+		if (!npmPackage.id().equalsIgnoreCase(packageId)) {
-
+			throw new InvalidRequestException(Msg.code(1297) + "Package ID " + npmPackage.id() + " doesn't match expected: " + packageId);
-		ourLog.info("Parsing package .tar.gz ({} bytes) from {}", bytes.length, theSourceDesc);
-
-		NpmPackage npmPackage = NpmPackage.fromPackage(new ByteArrayInputStream(bytes));
-		if (!npmPackage.id().equalsIgnoreCase(thePackageId)) {
-			throw new InvalidRequestException(Msg.code(1297) + "Package ID " + npmPackage.id() + " doesn't match expected: " + thePackageId);
 		}
-		if (!PackageVersionComparator.isEquivalent(thePackageVersionId, npmPackage.version())) {
+		if (!PackageVersionComparator.isEquivalent(initialPackageVersionId, npmPackage.version())) {
-			throw new InvalidRequestException(Msg.code(1298) + "Package ID " + npmPackage.version() + " doesn't match expected: " + thePackageVersionId);
+			throw new InvalidRequestException(Msg.code(1298) + "Package ID " + npmPackage.version() + " doesn't match expected: " + initialPackageVersionId);
 		}
 
 		String packageVersionId = npmPackage.version();
@@ -249,19 +271,18 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 		IBaseBinary binary = createPackageBinary(bytes);
 
 		return newTxTemplate().execute(tx -> {
-
 			ResourceTable persistedPackage = createResourceBinary(binary);
-			NpmPackageEntity pkg = myPackageDao.findByPackageId(thePackageId).orElseGet(() -> createPackage(npmPackage));
+			NpmPackageEntity pkg = myPackageDao.findByPackageId(packageId).orElseGet(() -> createPackage(npmPackage));
-			NpmPackageVersionEntity packageVersion = myPackageVersionDao.findByPackageIdAndVersion(thePackageId, packageVersionId).orElse(null);
+			NpmPackageVersionEntity packageVersion = myPackageVersionDao.findByPackageIdAndVersion(packageId, packageVersionId).orElse(null);
 			if (packageVersion != null) {
 				NpmPackage existingPackage = loadPackageFromCacheOnly(packageVersion.getPackageId(), packageVersion.getVersionId());
-				String msg = "Package version already exists in local storage, no action taken: " + thePackageId + "#" + packageVersionId;
+				String msg = "Package version already exists in local storage, no action taken: " + packageId + "#" + packageVersionId;
 				getProcessingMessages(existingPackage).add(msg);
 				ourLog.info(msg);
 				return existingPackage;
 			}
 
-			boolean currentVersion = updateCurrentVersionFlagForAllPackagesBasedOnNewIncomingVersion(thePackageId, packageVersionId);
+			boolean currentVersion = updateCurrentVersionFlagForAllPackagesBasedOnNewIncomingVersion(packageId, packageVersionId);
 			String packageDesc = null;
 			if (npmPackage.description() != null) {
 				if (npmPackage.description().length() > NpmPackageVersionEntity.PACKAGE_DESC_LENGTH) {
@@ -271,16 +292,16 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 				}
 			}
 			if (currentVersion) {
-				getProcessingMessages(npmPackage).add("Marking package " + thePackageId + "#" + thePackageVersionId + " as current version");
+				getProcessingMessages(npmPackage).add("Marking package " + packageId + "#" + initialPackageVersionId + " as current version");
 				pkg.setCurrentVersionId(packageVersionId);
 				pkg.setDescription(packageDesc);
 				myPackageDao.save(pkg);
 			} else {
-				getProcessingMessages(npmPackage).add("Package " + thePackageId + "#" + thePackageVersionId + " is not the newest version");
+				getProcessingMessages(npmPackage).add("Package " + packageId + "#" + initialPackageVersionId + " is not the newest version");
 			}
 
 			packageVersion = new NpmPackageVersionEntity();
-			packageVersion.setPackageId(thePackageId);
+			packageVersion.setPackageId(packageId);
 			packageVersion.setVersionId(packageVersionId);
 			packageVersion.setPackage(pkg);
 			packageVersion.setPackageBinary(persistedPackage);
@@ -353,7 +374,7 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 					String resType = packageContext.getResourceType(resource);
 					String msg = "Indexing " + resType + " Resource[" + dirName + '/' + nextFile + "] with URL: " + defaultString(url) + "|" + defaultString(version);
 					getProcessingMessages(npmPackage).add(msg);
-					ourLog.info("Package[{}#{}] " + msg, thePackageId, packageVersionId);
+					ourLog.info("Package[{}#{}] " + msg, packageId, packageVersionId);
 				}
 			}
 
@@ -361,7 +382,13 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 
 			return npmPackage;
 		});
+	}
 
+	@Override
+	public NpmPackage addPackageToCache(String thePackageId, String thePackageVersionId, InputStream thePackageTgzInputStream, String theSourceDesc) throws IOException {
+		NpmPackageData npmData = myPackageLoaderSvc.createNpmPackageDataFromData(thePackageId, thePackageVersionId, theSourceDesc, thePackageTgzInputStream);
+
+		return addPackageToCacheInternal(npmData);
 	}
 
 	private ResourceTable createResourceBinary(IBaseBinary theResourceBinary) {
@@ -426,24 +453,29 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 	@Override
 	@Transactional
 	public NpmPackage loadPackage(String thePackageId, String thePackageVersion) throws FHIRException, IOException {
+		// check package cache
 		NpmPackage cachedPackage = loadPackageFromCacheOnly(thePackageId, thePackageVersion);
 		if (cachedPackage != null) {
 			return cachedPackage;
 		}
 
-		InputStreamWithSrc pkg = super.loadFromPackageServer(thePackageId, thePackageVersion);
+		// otherwise we have to load it from packageloader
-		if (pkg == null) {
+		NpmPackageData pkgData = myPackageLoaderSvc.fetchPackageFromPackageSpec(thePackageId, thePackageVersion);
-			throw new ResourceNotFoundException(Msg.code(1301) + "Unable to locate package " + thePackageId + "#" + thePackageVersion);
-		}
 
 		try {
-			NpmPackage retVal = addPackageToCache(thePackageId, thePackageVersion == null ? pkg.version : thePackageVersion, pkg.stream, pkg.url);
+			// and add it to the cache
-			getProcessingMessages(retVal).add(0, "Package fetched from server at: " + pkg.url);
+			NpmPackage retVal = addPackageToCacheInternal(pkgData);
+			getProcessingMessages(retVal)
+				.add(0, "Package fetched from server at: " + pkgData.getPackage().url());
 			return retVal;
 		} finally {
-			pkg.stream.close();
+			pkgData.getInputStream().close();
 		}
+	}
 
+	@Override
+	public NpmPackage loadPackage(String theS) throws FHIRException, IOException {
+		return loadPackage(theS, null);
 	}
 
 	private TransactionTemplate newTxTemplate() {
@@ -458,7 +490,7 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 
 		String sourceDescription = "Embedded content";
 		if (isNotBlank(theInstallationSpec.getPackageUrl())) {
-			byte[] contents = loadPackageUrlContents(theInstallationSpec.getPackageUrl());
+			byte[] contents = myPackageLoaderSvc.loadPackageUrlContents(theInstallationSpec.getPackageUrl());
 			theInstallationSpec.setPackageContents(contents);
 			sourceDescription = theInstallationSpec.getPackageUrl();
 		}
@@ -476,33 +508,6 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac
 		});
 	}
 
-	protected byte[] loadPackageUrlContents(String thePackageUrl) {
-		if (thePackageUrl.startsWith("classpath:")) {
-			return ClasspathUtil.loadResourceAsByteArray(thePackageUrl.substring("classpath:" .length()));
-		} else if (thePackageUrl.startsWith("file:")) {
-			try {
-				byte[] bytes = Files.readAllBytes(Paths.get(new URI(thePackageUrl)));
-				return bytes;
-			} catch (IOException | URISyntaxException e) {
-				throw new InternalErrorException(Msg.code(2031) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
-			}
-		} else {
-			HttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
-			try (CloseableHttpResponse request = HttpClientBuilder
-				.create()
-				.setConnectionManager(connManager)
-				.build()
-				.execute(new HttpGet(thePackageUrl))) {
-				if (request.getStatusLine().getStatusCode() != 200) {
-					throw new ResourceNotFoundException(Msg.code(1303) + "Received HTTP " + request.getStatusLine().getStatusCode() + " from URL: " + thePackageUrl);
-				}
-				return IOUtils.toByteArray(request.getEntity().getContent());
-			} catch (IOException e) {
-				throw new InternalErrorException(Msg.code(1304) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
-			}
-		}
-	}
-
 	@Override
 	@Transactional
 	public IBaseResource loadPackageAssetByUrl(FhirVersionEnum theFhirVersion, String theCanonicalUrl) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/PackageInstallerSvcImpl.java

@@ -36,6 +36,7 @@ import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao;
 import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionEntity;
 import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
+import ca.uhn.fhir.jpa.packages.loader.PackageResourceParsingSvc;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistryController;
 import ca.uhn.fhir.jpa.searchparam.util.SearchParameterHelper;
@@ -43,7 +44,6 @@ import ca.uhn.fhir.rest.api.server.IBundleProvider;
 import ca.uhn.fhir.rest.param.StringParam;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.UriParam;
-import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
 import ca.uhn.fhir.util.FhirTerser;
 import ca.uhn.fhir.util.SearchParameterUtil;
@@ -67,12 +67,12 @@ import org.springframework.transaction.support.TransactionTemplate;
 import javax.annotation.Nonnull;
 import javax.annotation.PostConstruct;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
+import static ca.uhn.fhir.jpa.packages.util.PackageUtils.DEFAULT_INSTALL_TYPES;
 import static org.apache.commons.lang3.StringUtils.defaultString;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 
@@ -82,15 +82,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank;
 public class PackageInstallerSvcImpl implements IPackageInstallerSvc {
 
 	private static final Logger ourLog = LoggerFactory.getLogger(PackageInstallerSvcImpl.class);
-	public static List<String> DEFAULT_INSTALL_TYPES = Collections.unmodifiableList(Lists.newArrayList(
+
-		"NamingSystem",
-		"CodeSystem",
-		"ValueSet",
-		"StructureDefinition",
-		"ConceptMap",
-		"SearchParameter",
-		"Subscription"
-		));
 
 	boolean enabled = true;
 	@Autowired
@@ -113,6 +105,8 @@ public class PackageInstallerSvcImpl implements IPackageInstallerSvc {
 	private PartitionSettings myPartitionSettings;
 	@Autowired
 	private SearchParameterHelper mySearchParameterHelper;
+	@Autowired
+	private PackageResourceParsingSvc myPackageResourceParsingSvc;
 
 	/**
 	 * Constructor
@@ -220,11 +214,12 @@ public class PackageInstallerSvcImpl implements IPackageInstallerSvc {
 		int[] count = new int[installTypes.size()];
 
 		for (int i = 0; i < installTypes.size(); i++) {
-			Collection<IBaseResource> resources = parseResourcesOfType(installTypes.get(i), npmPackage);
+			String type = installTypes.get(i);
+
+			Collection<IBaseResource> resources = myPackageResourceParsingSvc.parseResourcesOfType(type, npmPackage);
 			count[i] = resources.size();
 
 			for (IBaseResource next : resources) {
-
 				try {
 					next = isStructureDefinitionWithoutSnapshot(next) ? generateSnapshot(next) : next;
 					create(next, theInstallationSpec, theOutcome);
@@ -307,24 +302,6 @@ public class PackageInstallerSvcImpl implements IPackageInstallerSvc {
 	 * ============================= Utility methods ===============================
 	 */
 
-	private List<IBaseResource> parseResourcesOfType(String type, NpmPackage pkg) {
-		if (!pkg.getFolders().containsKey("package")) {
-			return Collections.emptyList();
-		}
-		ArrayList<IBaseResource> resources = new ArrayList<>();
-		List<String> filesForType = pkg.getFolders().get("package").getTypes().get(type);
-		if (filesForType != null) {
-			for (String file : filesForType) {
-				try {
-					byte[] content = pkg.getFolders().get("package").fetchFile(file);
-					resources.add(myFhirContext.newJsonParser().parseResource(new String(content)));
-				} catch (IOException e) {
-					throw new InternalErrorException(Msg.code(1289) + "Cannot install resource of type " + type + ": Could not fetch file " + file, e);
-				}
-			}
-		}
-		return resources;
-	}
 
 	private void create(IBaseResource theResource, PackageInstallationSpec theInstallationSpec, PackageInstallOutcomeJson theOutcome) {
 		IFhirResourceDao dao = myDaoRegistry.getResourceDao(theResource.getClass());

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/loader/NpmPackageData.java

@@ -0,0 +1,78 @@
+package ca.uhn.fhir.jpa.packages.loader;
+
+import org.hl7.fhir.utilities.npm.NpmPackage;
+
+import java.io.InputStream;
+
+public class NpmPackageData {
+
+	/**
+	 * package id (npm id)
+	 */
+	private final String myPackageId;
+
+	/**
+	 * package version id (npm version)
+	 */
+	private final String myPackageVersionId;
+
+	/**
+	 * package description (url to find package, often)
+	 */
+	private final String mySourceDesc;
+
+	/**
+	 * The raw bytes of the entire package
+	 */
+	private final byte[] myBytes;
+
+	/**
+	 * The actual NpmPackage.
+	 */
+	private final NpmPackage myPackage;
+
+	/**
+	 * The raw stream of the entire npm package contents
+	 */
+	private final InputStream myInputStream;
+
+	public NpmPackageData(
+		String thePackageId,
+		String thePackageVersionId,
+		String theSourceDesc,
+		byte[] theBytes,
+		NpmPackage thePackage,
+		InputStream theStream
+	) {
+		myPackageId = thePackageId;
+		myPackageVersionId = thePackageVersionId;
+		mySourceDesc = theSourceDesc;
+		myBytes = theBytes;
+		myPackage = thePackage;
+		myInputStream = theStream;
+	}
+
+	public byte[] getBytes() {
+		return myBytes;
+	}
+
+	public NpmPackage getPackage() {
+		return myPackage;
+	}
+
+	public InputStream getInputStream() {
+		 return myInputStream;
+	}
+
+	public String getPackageId() {
+		return myPackageId;
+	}
+
+	public String getPackageVersionId() {
+		return myPackageVersionId;
+	}
+
+	public String getSourceDesc() {
+		return mySourceDesc;
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/loader/PackageLoaderSvc.java

@@ -0,0 +1,183 @@
+package ca.uhn.fhir.jpa.packages.loader;
+
+import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.jpa.packages.PackageInstallationSpec;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
+import ca.uhn.fhir.util.ClasspathUtil;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.Validate;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.utilities.npm.BasePackageCacheManager;
+import org.hl7.fhir.utilities.npm.NpmPackage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+public class PackageLoaderSvc extends BasePackageCacheManager {
+
+	private static final Logger ourLog = LoggerFactory.getLogger(PackageLoaderSvc.class);
+
+	public NpmPackageData fetchPackageFromPackageSpec(PackageInstallationSpec theSpec) throws IOException {
+		if (isNotBlank(theSpec.getPackageUrl())) {
+			byte[] contents = loadPackageUrlContents(theSpec.getPackageUrl());
+			return createNpmPackageDataFromData(
+				theSpec.getName(),
+				theSpec.getVersion(),
+				theSpec.getPackageUrl(),
+				new ByteArrayInputStream(contents)
+			);
+		}
+
+		return fetchPackageFromServerInternal(theSpec.getName(), theSpec.getVersion());
+	}
+
+	/**
+	 * Loads the package, but won't save it anywhere.
+	 * Returns the data to the caller
+	 *
+	 * @return - a POJO containing information about the NpmPackage, as well as it's contents
+	 * 			as fetched from the server
+	 * @throws IOException
+	 */
+	public NpmPackageData fetchPackageFromPackageSpec(
+		String thePackageId,
+		String thePackageVersion
+	) throws FHIRException, IOException {
+		return fetchPackageFromServerInternal(thePackageId, thePackageVersion);
+	}
+
+	private NpmPackageData fetchPackageFromServerInternal(
+		String thePackageId,
+		String thePackageVersion
+	) throws IOException {
+		BasePackageCacheManager.InputStreamWithSrc pkg = this.loadFromPackageServer(thePackageId, thePackageVersion);
+
+		if (pkg == null) {
+			throw new ResourceNotFoundException(Msg.code(1301) + "Unable to locate package " + thePackageId + "#" + thePackageVersion);
+		}
+
+		NpmPackageData npmPackage = createNpmPackageDataFromData(
+			thePackageId,
+			thePackageVersion == null ? pkg.version : thePackageVersion,
+			pkg.url,
+			pkg.stream
+		);
+
+		return npmPackage;
+	}
+
+	/**
+	 * Creates an NpmPackage data object.
+	 *
+	 * @param thePackageId - the id of the npm package
+	 * @param thePackageVersionId - the version id of the npm package
+	 * @param theSourceDesc - the installation spec description or package url
+	 * @param thePackageTgzInputStream - the package contents.
+	 *                                  Typically fetched from a server, but can be added directly to the package spec
+	 * @return
+	 * @throws IOException
+	 */
+	public NpmPackageData createNpmPackageDataFromData(
+		String thePackageId,
+		String thePackageVersionId,
+		String theSourceDesc,
+		InputStream thePackageTgzInputStream
+	) throws IOException {
+		Validate.notBlank(thePackageId, "thePackageId must not be null");
+		Validate.notBlank(thePackageVersionId, "thePackageVersionId must not be null");
+		Validate.notNull(thePackageTgzInputStream, "thePackageTgzInputStream must not be null");
+
+		byte[] bytes = IOUtils.toByteArray(thePackageTgzInputStream);
+
+		ourLog.info("Parsing package .tar.gz ({} bytes) from {}", bytes.length, theSourceDesc);
+
+		NpmPackage npmPackage = NpmPackage.fromPackage(new ByteArrayInputStream(bytes));
+
+		return new NpmPackageData(
+			thePackageId,
+			thePackageVersionId,
+			theSourceDesc,
+			bytes,
+			npmPackage,
+			thePackageTgzInputStream
+		);
+	}
+
+	@Override
+	public NpmPackage loadPackageFromCacheOnly(String theS, @Nullable String theS1) {
+		throw new UnsupportedOperationException(
+			Msg.code(2215)
+			+ "Cannot load from cache. "
+			+ "Caching not supported in PackageLoaderSvc. Use JpaPackageCache instead."
+		);
+	}
+
+	@Override
+	public NpmPackage addPackageToCache(String theS, String theS1, InputStream theInputStream, String theS2) throws IOException {
+		throw new UnsupportedOperationException(
+			Msg.code(2216)
+			+ "Cannot add to cache. "
+			+ "Caching not supported in PackageLoaderSvc. Use JpaPackageCache instead."
+		);
+	}
+
+	@Override
+	public NpmPackage loadPackage(String theS, String theS1) throws FHIRException {
+		/*
+		 * We throw an exception because while we could pipe this call through
+		 * to loadPackageOnly ourselves, returning NpmPackage details
+		 * on their own provides no value if nothing is cached/loaded onto hard disk somewhere
+		 *
+		 */
+		throw new UnsupportedOperationException(
+			Msg.code(2217)
+			+ "No packages are cached; "
+			+ " this service only loads from the server directly. "
+			+ "Call fetchPackageFromServer to fetch the npm package from the server. "
+			+ "Or use JpaPackageCache for a cache implementation."
+		);
+	}
+
+	public byte[] loadPackageUrlContents(String thePackageUrl) {
+		if (thePackageUrl.startsWith("classpath:")) {
+			return ClasspathUtil.loadResourceAsByteArray(thePackageUrl.substring("classpath:" .length()));
+		} else if (thePackageUrl.startsWith("file:")) {
+			try {
+				byte[] bytes = Files.readAllBytes(Paths.get(new URI(thePackageUrl)));
+				return bytes;
+			} catch (IOException | URISyntaxException e) {
+				throw new InternalErrorException(Msg.code(2031) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
+			}
+		} else {
+			HttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
+			try (CloseableHttpResponse request = HttpClientBuilder
+				.create()
+				.setConnectionManager(connManager)
+				.build()
+				.execute(new HttpGet(thePackageUrl))) {
+				if (request.getStatusLine().getStatusCode() != 200) {
+					throw new ResourceNotFoundException(Msg.code(1303) + "Received HTTP " + request.getStatusLine().getStatusCode() + " from URL: " + thePackageUrl);
+				}
+				return IOUtils.toByteArray(request.getEntity().getContent());
+			} catch (IOException e) {
+				throw new InternalErrorException(Msg.code(1304) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
+			}
+		}
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/loader/PackageResourceParsingSvc.java

@@ -0,0 +1,46 @@
+package ca.uhn.fhir.jpa.packages.loader;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.utilities.npm.NpmPackage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class PackageResourceParsingSvc {
+
+	private final FhirContext myFhirContext;
+
+	public PackageResourceParsingSvc(FhirContext theContext) {
+		myFhirContext = theContext;
+	}
+
+	/**
+	 * Parses out resource of theType from provided package
+	 * @param theType - the resource type
+	 * @param thePkg - the npm package
+	 * @return - a list of all resources that match type theType in package thePkg
+	 */
+	public List<IBaseResource> parseResourcesOfType(String theType, NpmPackage thePkg) {
+		if (!thePkg.getFolders().containsKey("package")) {
+			return Collections.emptyList();
+		}
+		ArrayList<IBaseResource> resources = new ArrayList<>();
+		List<String> filesForType = thePkg.getFolders().get("package").getTypes().get(theType);
+		if (filesForType != null) {
+			for (String file : filesForType) {
+				try {
+					byte[] content = thePkg.getFolders().get("package").fetchFile(file);
+					resources.add(myFhirContext.newJsonParser().parseResource(new String(content)));
+				} catch (IOException e) {
+					throw new InternalErrorException(Msg.code(1289) + "Cannot install resource of type " + theType + ": Could not fetch file " + file, e);
+				}
+			}
+		}
+		return resources;
+	}
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/util/PackageUtils.java

@@ -0,0 +1,24 @@
+package ca.uhn.fhir.jpa.packages.util;
+
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.List;
+
+public class PackageUtils {
+
+	public static final String LOADER_WITH_CACHE = "loaderWithCache";
+
+	/**
+	 * Default install types
+	 */
+	public static List<String> DEFAULT_INSTALL_TYPES = Collections.unmodifiableList(Lists.newArrayList(
+		"NamingSystem",
+		"CodeSystem",
+		"ValueSet",
+		"StructureDefinition",
+		"ConceptMap",
+		"SearchParameter",
+		"Subscription"
+	));
+}

hapi-fhir-jpaserver-elastic-test-utilities/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-mdm/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-model/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-searchparam/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-subscription/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-dstu2/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-dstu3/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-dstu3/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerDstu3Test.java

@@ -7,6 +7,7 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao;
 import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
 import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionEntity;
+import ca.uhn.fhir.jpa.packages.util.PackageUtils;
 import ca.uhn.fhir.jpa.test.BaseJpaDstu3Test;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
@@ -25,6 +26,7 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 
 import java.util.Date;
 import java.util.Optional;
@@ -51,7 +53,8 @@ public class IgInstallerDstu3Test extends BaseJpaDstu3Test {
 	@Autowired
 	private PackageInstallerSvcImpl igInstaller;
 	@Autowired
-	private IPackageCacheManager myPackageCacheManager;
+	@Qualifier(PackageUtils.LOADER_WITH_CACHE)
+	private IHapiPackageCacheManager myPackageCacheManager;
 	private Server myServer;
 	private FakeNpmServlet myFakeNpmServlet;
 	@Autowired

hapi-fhir-jpaserver-test-r4/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/packages/NpmR4Test.java

@@ -204,7 +204,10 @@ public class NpmR4Test extends BaseJpaR4Test {
 		byte[] bytes = ClasspathUtil.loadResourceAsByteArray("/packages/hl7.fhir.uv.shorthand-0.12.0.tgz");
 		myFakeNpmServlet.responses.put("/hl7.fhir.uv.shorthand/0.12.0", bytes);
 
-		PackageInstallationSpec spec = new PackageInstallationSpec().setName("hl7.fhir.uv.shorthand").setVersion("0.12.0").setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
+		PackageInstallationSpec spec = new PackageInstallationSpec()
+			.setName("hl7.fhir.uv.shorthand")
+			.setVersion("0.12.0")
+			.setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
 		PackageInstallOutcomeJson outcome = myPackageInstallerSvc.install(spec);
 		assertEquals(1, outcome.getResourcesInstalled().get("CodeSystem"));
 
@@ -409,7 +412,10 @@ public class NpmR4Test extends BaseJpaR4Test {
 
 		List<String> resourceList = new ArrayList<>();
 		resourceList.add("Organization");
-		PackageInstallationSpec spec = new PackageInstallationSpec().setName("test-organizations").setVersion("1.0.0").setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
+		PackageInstallationSpec spec = new PackageInstallationSpec()
+			.setName("test-organizations")
+			.setVersion("1.0.0")
+			.setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
 		spec.setInstallResourceTypes(resourceList);
 		PackageInstallOutcomeJson outcome = myPackageInstallerSvc.install(spec);
 		assertEquals(3, outcome.getResourcesInstalled().get("Organization"));
@@ -448,7 +454,10 @@ public class NpmR4Test extends BaseJpaR4Test {
 
 		List<String> resourceList = new ArrayList<>();
 		resourceList.add("Organization");
-		PackageInstallationSpec spec = new PackageInstallationSpec().setName("test-missing-identifier-package").setVersion("1.0.0").setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
+		PackageInstallationSpec spec = new PackageInstallationSpec()
+			.setName("test-missing-identifier-package")
+			.setVersion("1.0.0")
+			.setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
 		spec.setInstallResourceTypes(resourceList);
 		try {
 			PackageInstallOutcomeJson outcome = myPackageInstallerSvc.install(spec);
@@ -471,7 +480,10 @@ public class NpmR4Test extends BaseJpaR4Test {
 
 		List<String> resourceList = new ArrayList<>();
 		resourceList.add("ImplementationGuide");
-		PackageInstallationSpec spec = new PackageInstallationSpec().setName("test-ig").setVersion("1.0.0").setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
+		PackageInstallationSpec spec = new PackageInstallationSpec()
+			.setName("test-ig")
+			.setVersion("1.0.0")
+			.setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
 		spec.setInstallResourceTypes(resourceList);
 		PackageInstallOutcomeJson outcome = myPackageInstallerSvc.install(spec);
 		ourLog.info("Outcome: {}", outcome);
@@ -499,7 +511,6 @@ public class NpmR4Test extends BaseJpaR4Test {
 		PackageInstallationSpec spec = new PackageInstallationSpec().setName("hl7.fhir.uv.onlydrafts").setVersion("0.11.1").setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL);
 		PackageInstallOutcomeJson outcome = myPackageInstallerSvc.install(spec);
 		assertEquals(0, outcome.getResourcesInstalled().size(), outcome.getResourcesInstalled().toString());
-
 	}
 
 	@Test
@@ -547,10 +558,8 @@ public class NpmR4Test extends BaseJpaR4Test {
 		// Ensure that we loaded the contents
 		IBundleProvider searchResult = myCodeSystemDao.search(SearchParameterMap.newSynchronous("url", new UriParam("http://hl7.org/fhir/uv/shorthand/CodeSystem/shorthand-code-system")));
 		assertEquals(1, searchResult.sizeOrThrowNpe());
-
 	}
 
-
 	@Test
 	public void testInstallR4PackageWithNoDescription() throws Exception {
 		myDaoConfig.setAllowExternalReferences(true);
@@ -597,7 +606,6 @@ public class NpmR4Test extends BaseJpaR4Test {
 
 	}
 
-
 	@Test
 	public void testLoadPackageUsingImpreciseId() throws Exception {
 		myDaoConfig.setAllowExternalReferences(true);

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/packages/loader/PackageLoaderSvcIT.java

@@ -0,0 +1,138 @@
+package ca.uhn.fhir.jpa.packages.loader;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.packages.FakeNpmServlet;
+import ca.uhn.fhir.jpa.packages.util.PackageUtils;
+import ca.uhn.fhir.test.utilities.JettyUtil;
+import ca.uhn.fhir.util.ClasspathUtil;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.utilities.npm.NpmPackage;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class PackageLoaderSvcIT {
+
+	@Spy
+	private FhirContext myFhirContext = FhirContext.forR4Cached();
+
+	private Server myServer;
+
+	private FakeNpmServlet myFakeNpmServlet;
+
+	private PackageLoaderSvc myPackageLoaderSvc;
+
+	private PackageResourceParsingSvc myResourceParsingSvc;
+
+	@BeforeEach
+	public void before() throws Exception {
+		myPackageLoaderSvc = new PackageLoaderSvc();
+		myResourceParsingSvc = new PackageResourceParsingSvc(myFhirContext);
+
+		myServer = new Server(0);
+		ServletHandler proxyHandler = new ServletHandler();
+		myFakeNpmServlet = new FakeNpmServlet();
+		ServletHolder servletHolder = new ServletHolder(myFakeNpmServlet);
+		proxyHandler.addServletWithMapping(servletHolder, "/*");
+		myServer.setHandler(proxyHandler);
+		myServer.start();
+
+		int port = JettyUtil.getPortForStartedServer(myServer);
+		myPackageLoaderSvc.getPackageServers().clear();
+		myPackageLoaderSvc.addPackageServer("http://localhost:" + port);
+
+		myFakeNpmServlet.getResponses().clear();
+	}
+
+	@AfterEach
+	public void after() throws Exception {
+		JettyUtil.closeServer(myServer);
+	}
+
+	@Test
+	public void fetchPackageFromServer_thenParseoutResources_inMemory() throws IOException {
+		// setup
+		String id = "test-exchange.fhir.us.com/2.1.1";
+		String versionId = "2.1.";
+		// this package has SearchParameters in it
+		byte[] bytes = ClasspathUtil.loadResourceAsByteArray("/packages/test-exchange-sample.tgz");
+		myFakeNpmServlet.getResponses().put(String.format("/%s/%s", id, versionId), bytes);
+
+		// test fetch from server by id and version
+		NpmPackageData result = myPackageLoaderSvc.fetchPackageFromPackageSpec(id, versionId);
+
+		// verify fetched data
+		assertNotNull(result);
+		assertNotNull(result.getPackage());
+		NpmPackage npmPackage = result.getPackage();
+
+		// test parse resources
+		List<IBaseResource> resources = new ArrayList<>();
+		List<String> resourcesToParse = PackageUtils.DEFAULT_INSTALL_TYPES;
+		for (String resourceType : resourcesToParse) {
+			resources.addAll(
+				myResourceParsingSvc.parseResourcesOfType(resourceType, npmPackage)
+			);
+		}
+
+		// verify fetched resources
+		assertFalse(resources.isEmpty());
+		assertEquals(1, resources.size());
+		assertEquals("SearchParameter", resources.get(0).fhirType());
+	}
+
+	/**
+	 * PackageLoaderSvc extends BasePackageCacheManger.
+	 * However, we do not want this service to have any
+	 * DAO dependence (ie, no cache).
+	 *
+	 * But since BasePackageCacheManger is in a different
+	 * codebase, we cannot remove some methods and must just
+	 * not support them.
+	 *
+	 * We'll test to make sure these stay unsupported
+	 * (barring a breakup of BasePackageCacheManager itself)
+	 */
+	@Test
+	public void anyCacheUtilizingMethod_throwsUnsupported() throws IOException {
+		// loadPackageFromCacheOnly
+		try {
+			myPackageLoaderSvc.loadPackageFromCacheOnly("id", "versionId");
+			fail();
+		} catch (UnsupportedOperationException ex) {
+			assertTrue(ex.getMessage().contains("Cannot load from cache."));
+		}
+
+		// addPackageToCache
+		try {
+			myPackageLoaderSvc.addPackageToCache("id", "version", Mockito.mock(InputStream.class), "description or url");
+			fail();
+		} catch (UnsupportedOperationException ex) {
+			assertTrue(ex.getMessage().contains("Cannot add to cache."));
+		}
+
+		// loadPackage
+		try {
+			myPackageLoaderSvc.loadPackage("id", "version");
+			fail();
+		} catch (UnsupportedOperationException ex) {
+			assertTrue(ex.getMessage().contains("No packages are cached;"));
+		}
+	}
+}

hapi-fhir-jpaserver-test-r4b/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-r5/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-utilities/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/packages/FakeNpmServlet.java

@@ -32,7 +32,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-class FakeNpmServlet extends HttpServlet {
+public class FakeNpmServlet extends HttpServlet {
 	private static final Logger ourLog = LoggerFactory.getLogger(FakeNpmServlet.class);
 
 	final Map<String, byte[]> responses = new HashMap<>();

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestDstu3Config.java

@@ -24,6 +24,7 @@ import ca.uhn.fhir.batch2.jobs.config.Batch2JobsConfig;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.batch2.JpaBatch2Config;
 import ca.uhn.fhir.jpa.config.HapiJpaConfig;
+import ca.uhn.fhir.jpa.config.PackageLoaderConfig;
 import ca.uhn.fhir.jpa.config.dstu3.JpaDstu3Config;
 import ca.uhn.fhir.jpa.config.util.HapiEntityManagerFactoryUtil;
 import ca.uhn.fhir.jpa.model.dialect.HapiFhirH2Dialect;
@@ -60,6 +61,7 @@ import static org.junit.jupiter.api.Assertions.fail;
 @Configuration
 @Import({
 	JpaDstu3Config.class,
+	PackageLoaderConfig.class,
 	HapiJpaConfig.class,
 	TestJPAConfig.class,
 	JpaBatch2Config.class,

hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java

@@ -26,6 +26,7 @@ import ca.uhn.fhir.jpa.batch2.JpaBatch2Config;
 import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
 import ca.uhn.fhir.jpa.binstore.MemoryBinaryStorageSvcImpl;
 import ca.uhn.fhir.jpa.config.HapiJpaConfig;
+import ca.uhn.fhir.jpa.config.PackageLoaderConfig;
 import ca.uhn.fhir.jpa.config.r4.JpaR4Config;
 import ca.uhn.fhir.jpa.config.util.HapiEntityManagerFactoryUtil;
 import ca.uhn.fhir.jpa.model.dialect.HapiFhirH2Dialect;
@@ -63,6 +64,7 @@ import static org.junit.jupiter.api.Assertions.fail;
 @Configuration
 @Import({
 	JpaR4Config.class,
+	PackageLoaderConfig.class,
 	HapiJpaConfig.class,
 	TestJPAConfig.class,
 	TestHSearchAddInConfig.DefaultLuceneHeap.class,

hapi-fhir-jpaserver-uhnfhirtest/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server-mdm/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server-openapi/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-server/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<artifactId>hapi-fhir-serviceloaders</artifactId>
 		<groupId>ca.uhn.hapi.fhir</groupId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<artifactId>hapi-fhir-serviceloaders</artifactId>
 		<groupId>ca.uhn.hapi.fhir</groupId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 
@@ -20,7 +20,7 @@
 		<dependency>
 			<groupId>ca.uhn.hapi.fhir</groupId>
 			<artifactId>hapi-fhir-caching-api</artifactId>
-			<version>6.3.8-SNAPSHOT</version>
+			<version>6.3.9-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>com.github.ben-manes.caffeine</groupId>

hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<artifactId>hapi-fhir-serviceloaders</artifactId>
 		<groupId>ca.uhn.hapi.fhir</groupId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<artifactId>hapi-fhir</artifactId>
 		<groupId>ca.uhn.hapi.fhir</groupId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-serviceloaders/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<artifactId>hapi-fhir</artifactId>
 		<groupId>ca.uhn.hapi.fhir</groupId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-client-apache</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-client-okhttp</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot-samples</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-sample-server-jersey</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir-spring-boot</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>hapi-fhir-spring-boot-samples</artifactId>

hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml

@@ -5,7 +5,7 @@
    <parent>
       <groupId>ca.uhn.hapi.fhir</groupId>
       <artifactId>hapi-deployable-pom</artifactId>
-      <version>6.3.8-SNAPSHOT</version>
+      <version>6.3.9-SNAPSHOT</version>
       <relativePath>../../hapi-deployable-pom/pom.xml</relativePath>
    </parent>
 

hapi-fhir-spring-boot/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-sql-migrate/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-sql-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java

@@ -32,6 +32,12 @@ public class AddColumnTask extends BaseTableColumnTypeTask {
 
 	private static final Logger ourLog = LoggerFactory.getLogger(AddColumnTask.class);
 
+	public AddColumnTask() {
+		this(null, null);
+		setDryRun(true);
+		myCheckForExistingTables = false;
+	}
+
 	public AddColumnTask(String theProductVersion, String theSchemaVersion) {
 		super(theProductVersion, theSchemaVersion);
 	}
@@ -44,10 +50,12 @@ public class AddColumnTask extends BaseTableColumnTypeTask {
 
 	@Override
 	public void doExecute() throws SQLException {
-		Set<String> columnNames = JdbcUtils.getColumnNames(getConnectionProperties(), getTableName());
+		if (myCheckForExistingTables) {
-		if (columnNames.contains(getColumnName())) {
+			Set<String> columnNames = JdbcUtils.getColumnNames(getConnectionProperties(), getTableName());
-			logInfo(ourLog, "Column {} already exists on table {} - No action performed", getColumnName(), getTableName());
+			if (columnNames.contains(getColumnName())) {
-			return;
+				logInfo(ourLog, "Column {} already exists on table {} - No action performed", getColumnName(), getTableName());
+				return;
+			}
 		}
 
 		String typeStatement = getTypeStatement();
@@ -82,7 +90,11 @@ public class AddColumnTask extends BaseTableColumnTypeTask {
 		if (isNullable()) {
 			nullable = "";
 		}
-		return type + " " + nullable;
+		if (myPrettyPrint) {
+			nullable = nullable.trim();
+		}
+		String space = isNullable() ? "" : " ";
+		return type + space + nullable;
 	}
 
 }

hapi-fhir-sql-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddTableByColumnTask.java

@@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.migrate.taskdef;
  * #L%
  */
 
+import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
 import ca.uhn.fhir.jpa.migrate.JdbcUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.builder.EqualsBuilder;
@@ -35,8 +36,15 @@ public class AddTableByColumnTask extends BaseTableTask {
 
 	private static final Logger ourLog = LoggerFactory.getLogger(AddTableByColumnTask.class);
 
-	private List<AddColumnTask> myAddColumnTasks = new ArrayList<>();
+	private final List<AddColumnTask> myAddColumnTasks = new ArrayList<>();
 	private List<String> myPkColumns;
+	private final List<ForeignKeyContainer> myFKColumns = new ArrayList<>();
+
+	public AddTableByColumnTask() {
+		this(null, null);
+		setDryRun(true);
+		myCheckForExistingTables = false;
+	}
 
 	public AddTableByColumnTask(String theProductVersion, String theSchemaVersion) {
 		super(theProductVersion, theSchemaVersion);
@@ -57,45 +65,98 @@ public class AddTableByColumnTask extends BaseTableTask {
 		myPkColumns = thePkColumns;
 	}
 
-	@Override
+	public void addForeignKey(ForeignKeyContainer theForeignKeyContainer) {
-	public void doExecute() throws SQLException {
+		myFKColumns.add(theForeignKeyContainer);
+	}
 
-		if (JdbcUtils.getTableNames(getConnectionProperties()).contains(getTableName())) {
+	public List<String> getPkColumns() {
-			logInfo(ourLog, "Already have table named {} - No action performed", getTableName());
+		return myPkColumns;
-			return;
+	}
-		}
 
+	public String generateSQLCreateScript() {
 		StringBuilder sb = new StringBuilder();
 		sb.append("CREATE TABLE ");
 		sb.append(getTableName());
-		sb.append(" ( ");
+		sb.append(" (");
+		if (myPrettyPrint) {
+			sb.append("\n");
+		} else {
+			sb.append(" ");
+		}
 
 		for (AddColumnTask next : myAddColumnTasks) {
 			next.setDriverType(getDriverType());
 			next.setTableName(getTableName());
 			next.validate();
 
+			if (myPrettyPrint) {
+				sb.append("\t");
+			}
+
 			sb.append(next.getColumnName());
 			sb.append(" ");
 			sb.append(next.getTypeStatement());
-			sb.append(", ");
+			sb.append(",");
+			if (myPrettyPrint) {
+				sb.append("\n");
+			} else {
+				sb.append(" ");
+			}
 		}
 
-		sb.append(" PRIMARY KEY (");
+		// primary keys
+		if (myPrettyPrint) {
+			sb.append("\t");
+		} else {
+			sb.append(" ");
+		}
+		sb.append("PRIMARY KEY (");
 		for (int i = 0; i < myPkColumns.size(); i++) {
 			if (i > 0) {
 				sb.append(", ");
 			}
 			sb.append(myPkColumns.get(i));
 		}
+
+		boolean hasForeignKeys = !myFKColumns.isEmpty();
+
+		sb.append(")");
+		if (hasForeignKeys) {
+			sb.append(",");
+		}
+		if (myPrettyPrint) {
+			sb.append("\n");
+		} else {
+			sb.append(" ");
+		}
+
+		DriverTypeEnum sqlEngine = getDriverType();
+
+		// foreign keys
+		if (!myFKColumns.isEmpty()) {
+			for (int i =0; i < myFKColumns.size(); i++) {
+				if (i > 0) {
+					sb.append(", ");
+				}
+				ForeignKeyContainer fk = myFKColumns.get(i);
+				if (myPrettyPrint) {
+					sb.append("\t");
+				}
+				sb.append(fk.generateSQL(sqlEngine, myPrettyPrint));
+				if (myPrettyPrint) {
+					sb.append("\n");
+				} else {
+					sb.append(" ");
+				}
+			}
+		}
+
 		sb.append(")");
 
-		sb.append(" ) ");
+		switch (sqlEngine) {
-
-		switch (getDriverType()) {
 			case MARIADB_10_1:
 			case MYSQL_5_7:
-				sb.append("engine=InnoDB");
+				sb.append(" engine=InnoDB");
 				break;
 			case DERBY_EMBEDDED:
 			case POSTGRES_9_4:
@@ -106,7 +167,17 @@ public class AddTableByColumnTask extends BaseTableTask {
 				break;
 		}
 
-		executeSql(getTableName(), sb.toString());
+		return sb.toString();
+	}
+
+	@Override
+	public void doExecute() throws SQLException {
+		if (myCheckForExistingTables && JdbcUtils.getTableNames(getConnectionProperties()).contains(getTableName())) {
+			logInfo(ourLog, "Already have table named {} - No action performed", getTableName());
+			return;
+		}
+
+		executeSql(getTableName(), generateSQLCreateScript());
 
 	}
 

hapi-fhir-sql-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java

@@ -73,6 +73,16 @@ public abstract class BaseTask {
 	private boolean myNoColumnShrink;
 	private boolean myFailureAllowed;
 	private boolean myRunDuringSchemaInitialization;
+	/**
+	 * Whether or not to check for existing tables
+	 * before generating SQL
+	 */
+	protected boolean myCheckForExistingTables = true;
+
+	/**
+	 * Whether or not to generate the SQL in a 'readable format'
+	 */
+	protected boolean myPrettyPrint = false;
 
 	protected BaseTask(String theProductVersion, String theSchemaVersion) {
 		myProductVersion = theProductVersion;
@@ -83,6 +93,10 @@ public abstract class BaseTask {
 		return myRunDuringSchemaInitialization;
 	}
 
+	public void setPrettyPrint(boolean thePrettyPrint) {
+		myPrettyPrint = thePrettyPrint;
+	}
+
 	/**
 	 * Should this task run even if we're doing the very first initialization of an empty schema. By
 	 * default we skip most tasks during that pass, since they just take up time and the

hapi-fhir-sql-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ForeignKeyContainer.java

@@ -0,0 +1,107 @@
+package ca.uhn.fhir.jpa.migrate.taskdef;
+
+import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
+
+import javax.annotation.Nonnull;
+
+public class ForeignKeyContainer {
+
+	/**
+	 * The parent table name
+	 */
+	private String myParentTableName;
+
+	/**
+	 * The name of the column in this table that holds the foreign key
+	 */
+	private String myColumnName;
+
+	/**
+	 * The column data type
+	 */
+	private ColumnTypeEnum myColumnTypeEnum;
+
+	/**
+	 * The name of the column in the parent table (that is the foreign key)
+	 */
+	private String myParentTableColumnName;
+
+	public ForeignKeyContainer(
+		String theColumnName,
+		ColumnTypeEnum theColumnTypeEnum,
+		String theParentTableName,
+		String theParentTableColumnName
+	) {
+		myColumnName = theColumnName;
+		myColumnTypeEnum = theColumnTypeEnum;
+		myParentTableName = theParentTableName;
+		myParentTableColumnName = theParentTableColumnName;
+	}
+
+	public String getParentTableName() {
+		return myParentTableName;
+	}
+
+	public void setParentTableName(String theParentTableName) {
+		myParentTableName = theParentTableName;
+	}
+
+	public String getColumnName() {
+		return myColumnName;
+	}
+
+	public void setColumnName(String theColumnName) {
+		myColumnName = theColumnName;
+	}
+
+	public String getParentTableColumnName() {
+		return myParentTableColumnName;
+	}
+
+	public void setParentTableColumnName(String theParentTableColumnName) {
+		myParentTableColumnName = theParentTableColumnName;
+	}
+
+	public ColumnTypeEnum getColumnTypeEnum() {
+		return myColumnTypeEnum;
+	}
+
+	public void setColumnTypeEnum(ColumnTypeEnum theColumnTypeEnum) {
+		myColumnTypeEnum = theColumnTypeEnum;
+	}
+
+	public String generateSQL(
+		@Nonnull DriverTypeEnum theDriverTypeEnum,
+		boolean thePrettyPrint
+	) {
+		switch (theDriverTypeEnum) {
+			case MYSQL_5_7:
+				return String.format(
+					"FOREIGN KEY (%s) REFERENCES %s(%s)",
+					myColumnName,
+					myParentTableName,
+					myParentTableColumnName
+				);
+			case MSSQL_2012:
+			case ORACLE_12C:
+				return String.format(
+					"%s %s FOREIGN KEY REFERENCES %s(%s)",
+					myColumnName,
+					myColumnTypeEnum.name(),
+					myParentTableName,
+					myParentTableColumnName
+				);
+			case POSTGRES_9_4:
+				return String.format(
+					"FOREIGN KEY(%s) REFERENCES %s(%s)",
+					myColumnName,
+					myParentTableName,
+					myParentTableColumnName
+				);
+			default:
+				throw new UnsupportedOperationException(
+					Msg.code(2232) + " SQL Engine " + theDriverTypeEnum.name() + " not supported for foreign key!");
+		}
+	}
+}

hapi-fhir-storage-batch2-jobs/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-storage-batch2/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-storage-cr/pom.xml

@@ -7,7 +7,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-storage-mdm/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-storage-test-utilities/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>

hapi-fhir-storage/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu2.1/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu2/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-dstu3/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-hl7org-dstu2/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-r4/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-r4b/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-structures-r5/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-test-utilities/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-testpage-overlay/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-dstu2.1/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-dstu2/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-dstu3/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-r4/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation-resources-r5/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-fhir-validation/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-deployable-pom</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
 	</parent>
 

hapi-tinder-plugin/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

hapi-tinder-test/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

pom.xml

@@ -6,7 +6,7 @@
 	<groupId>ca.uhn.hapi.fhir</groupId>
 	<artifactId>hapi-fhir</artifactId>
 	<packaging>pom</packaging>
-	<version>6.3.8-SNAPSHOT</version>
+	<version>6.3.9-SNAPSHOT</version>
 	<name>HAPI-FHIR</name>
 	<description>An open-source implementation of the FHIR specification in Java.</description>
 	<url>https://hapifhir.io</url>
@@ -2117,7 +2117,7 @@
                             <groupId>ca.uhn.hapi.fhir</groupId>
                             <artifactId>hapi-fhir-checkstyle</artifactId>
 									<!-- Remember to bump this when you upgrade the version -->
-                            <version>6.3.8-SNAPSHOT</version>
+                            <version>6.3.9-SNAPSHOT</version>
                         </dependency>
 					</dependencies>
 				</plugin>

tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 

tests/hapi-fhir-base-test-mindeps-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 

tests/hapi-fhir-base-test-mindeps-server/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
 		<artifactId>hapi-fhir</artifactId>
-		<version>6.3.8-SNAPSHOT</version>
+		<version>6.3.9-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>