More bug-hunting. DSTU3 was missing a null check.

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java

@@ -93,12 +93,12 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 
 		boolean allSystemsAreSuppportedByTerminologyService = true;
 		for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
-			if (!myTerminologySvc.supportsSystem(next.getSystem())) {
+			if (!isBlank(next.getSystem()) && !myTerminologySvc.supportsSystem(next.getSystem())) {
 				allSystemsAreSuppportedByTerminologyService = false;
 			}
 		}
 		for (ConceptSetComponent next : theSource.getCompose().getExclude()) {
-			if (!myTerminologySvc.supportsSystem(next.getSystem())) {
+			if (!isBlank(next.getSystem()) && !myTerminologySvc.supportsSystem(next.getSystem())) {
 				allSystemsAreSuppportedByTerminologyService = false;
 			}
 		}
@@ -125,12 +125,12 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 
 		boolean allSystemsAreSuppportedByTerminologyService = true;
 		for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
-			if (!myTerminologySvc.supportsSystem(next.getSystem())) {
+			if (!isBlank(next.getSystem()) && !myTerminologySvc.supportsSystem(next.getSystem())) {
 				allSystemsAreSuppportedByTerminologyService = false;
 			}
 		}
 		for (ConceptSetComponent next : theSource.getCompose().getExclude()) {
-			if (!myTerminologySvc.supportsSystem(next.getSystem())) {
+			if (!isBlank(next.getSystem()) && !myTerminologySvc.supportsSystem(next.getSystem())) {
 				allSystemsAreSuppportedByTerminologyService = false;
 			}
 		}
@@ -160,6 +160,7 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 		}
 
 		ValueSet source = new ValueSet();
+		source.setUrl(theUri);
 
 		source.getCompose().addInclude().addValueSet(theUri);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoValueSetR4.java

@@ -156,6 +156,7 @@ public class FhirResourceDaoValueSetR4 extends FhirResourceDaoR4<ValueSet> imple
 		}
 
 		ValueSet source = new ValueSet();
+		source.setUrl(theUri);
 
 		source.getCompose().addInclude().addValueSet(theUri);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoValueSetR5.java

@@ -162,6 +162,7 @@ public class FhirResourceDaoValueSetR5 extends FhirResourceDaoR5<ValueSet> imple
 		}
 
 		ValueSet source = new ValueSet();
+		source.setUrl(theUri);
 
 		source.getCompose().addInclude().addValueSet(theUri);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java

@@ -210,6 +210,7 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvcImpl implemen
 
 	@Override
 	public List<VersionIndependentConcept> expandValueSet(String theValueSet) {
+		// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
 		ValueSet vs = myValidationSupport.fetchResource(myContext, ValueSet.class, theValueSet);
 		if (vs == null) {
 			super.throwInvalidValueSet(theValueSet);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR4.java

@@ -122,6 +122,7 @@ public class HapiTerminologySvcR4 extends BaseHapiTerminologySvcImpl implements
 
 	@Override
 	public List<VersionIndependentConcept> expandValueSet(String theValueSet) {
+		// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
 		ValueSet vs = myValidationSupport.fetchResource(myContext, ValueSet.class, theValueSet);
 		if (vs == null) {
 			super.throwInvalidValueSet(theValueSet);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcR5.java

@@ -131,6 +131,7 @@ public class HapiTerminologySvcR5 extends BaseHapiTerminologySvcImpl implements
 
 	@Override
 	public List<VersionIndependentConcept> expandValueSet(String theValueSet) {
+		// TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
 		ValueSet valueSetR5 = myValidationSupport.fetchResource(myContext, ValueSet.class, theValueSet);
 		if (valueSetR5 == null) {
 			super.throwInvalidValueSet(theValueSet);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3ValueSetTest.java

@@ -1,14 +1,16 @@
 package ca.uhn.fhir.jpa.provider.dstu3;
 
+import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.IFhirResourceDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
 import ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoR4TerminologyTest;
-import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
 import ca.uhn.fhir.jpa.entity.TermConcept;
 import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
 import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.util.TestUtil;
 import ca.uhn.fhir.util.UrlUtil;
@@ -25,11 +27,14 @@ import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode;
 import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.FilterOperator;
+import org.hl7.fhir.dstu3.model.codesystems.HttpVerb;
 import org.hl7.fhir.instance.model.api.IIdType;
+import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.Test;
-import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallbackWithoutResult;
+import org.springframework.transaction.support.TransactionTemplate;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -43,18 +48,86 @@ import static org.junit.Assert.*;
 public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3Test {
 
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(ResourceProviderDstu3ValueSetTest.class);
+	private IIdType myExtensionalCsId;
 	private IIdType myExtensionalVsId;
 	private IIdType myLocalValueSetId;
+	private Long myExtensionalCsIdOnResourceTable;
+	private Long myExtensionalVsIdOnResourceTable;
 	private ValueSet myLocalVs;
 
-	@Before
+//	@Before
-	@Transactional
+//	@Transactional
-	public void before02() throws IOException {
+//	public void before02() throws IOException {
-		CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml");
+//		CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml");
-		myCodeSystemDao.create(cs, mySrd);
+//		myCodeSystemDao.create(cs, mySrd);
+//
+//		ValueSet upload = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
+//		myExtensionalVsId = myValueSetDao.create(upload, mySrd).getId().toUnqualifiedVersionless();
+//	}
 
-		ValueSet upload = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
+	private void loadAndPersistCodeSystemAndValueSet(HttpVerb theVerb) throws IOException {
-		myExtensionalVsId = myValueSetDao.create(upload, mySrd).getId().toUnqualifiedVersionless();
+		loadAndPersistCodeSystem(theVerb);
+		loadAndPersistValueSet(theVerb);
+	}
+
+	private void loadAndPersistCodeSystem(HttpVerb theVerb) throws IOException {
+		CodeSystem codeSystem = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml");
+		codeSystem.setId("CodeSystem/cs");
+		persistCodeSystem(codeSystem, theVerb);
+	}
+
+	private void persistCodeSystem(CodeSystem theCodeSystem, HttpVerb theVerb) {
+		switch (theVerb) {
+			case POST:
+				new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+					@Override
+					protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
+						myExtensionalCsId = myCodeSystemDao.create(theCodeSystem, mySrd).getId().toUnqualifiedVersionless();
+					}
+				});
+				break;
+			case PUT:
+				new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+					@Override
+					protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
+						myExtensionalCsId = myCodeSystemDao.update(theCodeSystem, mySrd).getId().toUnqualifiedVersionless();
+					}
+				});
+				break;
+			default:
+				throw new IllegalArgumentException("HTTP verb is not supported: " + theVerb);
+		}
+		myExtensionalCsIdOnResourceTable = myCodeSystemDao.readEntity(myExtensionalCsId, null).getId();
+	}
+
+	private void loadAndPersistValueSet(HttpVerb theVerb) throws IOException {
+		ValueSet valueSet = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
+		valueSet.setId("ValueSet/vs");
+		persistValueSet(valueSet, theVerb);
+	}
+
+	private void persistValueSet(ValueSet theValueSet, HttpVerb theVerb) {
+		switch (theVerb) {
+			case POST:
+				new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+					@Override
+					protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
+						myExtensionalVsId = myValueSetDao.create(theValueSet, mySrd).getId().toUnqualifiedVersionless();
+					}
+				});
+				break;
+			case PUT:
+				new TransactionTemplate(myTxManager).execute(new TransactionCallbackWithoutResult() {
+					@Override
+					protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
+						myExtensionalVsId = myValueSetDao.update(theValueSet, mySrd).getId().toUnqualifiedVersionless();
+					}
+				});
+				break;
+			default:
+				throw new IllegalArgumentException("HTTP verb is not supported: " + theVerb);
+		}
+		myExtensionalVsIdOnResourceTable = myValueSetDao.readEntity(myExtensionalVsId, null).getId();
 	}
 
 	private CodeSystem createExternalCs() {
@@ -169,7 +242,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 
 	@Test
-	public void testExpandValueSetPropertySearchWithRegexExcludeUsingOr() {
+	public void testExpandValueSetPropertySearchWithRegexExcludeUsingOr() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		createLoincSystemWithSomeCodes();
 
 		List<String> codes;
@@ -207,7 +282,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 
 	@Test
-	public void testExpandValueSetPropertySearchWithRegexExcludeNoFilter() {
+	public void testExpandValueSetPropertySearchWithRegexExcludeNoFilter() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		createLoincSystemWithSomeCodes();
 
 		List<String> codes;
@@ -233,7 +310,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 
 	@Test
-	public void testExpandById() {
+	public void testExpandById() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		//@formatter:off
 		Parameters respParam = ourClient
 			.operation()
@@ -263,7 +342,41 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	}
 
 	@Test
-	public void testExpandByIdWithFilter() {
+	public void testExpandByIdWithPreExpansion() throws Exception {
+		myDaoConfig.setPreExpandValueSetsExperimental(true);
+
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+
+		Parameters respParam = ourClient
+			.operation()
+			.onInstance(myExtensionalVsId)
+			.named("expand")
+			.withNoParameters(Parameters.class)
+			.execute();
+		ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
+
+		String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
+		ourLog.info(resp);
+		assertThat(resp, Matchers.containsString("<ValueSet xmlns=\"http://hl7.org/fhir\">"));
+		assertThat(resp, Matchers.containsString("<expansion>"));
+		assertThat(resp, Matchers.containsString("<contains>"));
+		assertThat(resp, Matchers.containsString("<system value=\"http://acme.org\"/>"));
+		assertThat(resp, Matchers.containsString("<code value=\"8450-9\"/>"));
+		assertThat(resp, Matchers.containsString("<display value=\"Systolic blood pressure--expiration\"/>"));
+		assertThat(resp, Matchers.containsString("</contains>"));
+		assertThat(resp, Matchers.containsString("<contains>"));
+		assertThat(resp, Matchers.containsString("<system value=\"http://acme.org\"/>"));
+		assertThat(resp, Matchers.containsString("<code value=\"11378-7\"/>"));
+		assertThat(resp, Matchers.containsString("<display value=\"Systolic blood pressure at First encounter\"/>"));
+		assertThat(resp, Matchers.containsString("</contains>"));
+		assertThat(resp, Matchers.containsString("</expansion>"));
+
+	}
+
+	@Test
+	public void testExpandByIdWithFilter() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
 
 		//@formatter:off
 		Parameters respParam = ourClient
@@ -290,7 +403,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	 * https://groups.google.com/d/msgid/hapi-fhir/CAN2Cfy8kW%2BAOkgC6VjPsU3gRCpExCNZBmJdi-k5R_TWeyWH4tA%40mail.gmail.com?utm_medium=email&utm_source=footer
 	 */
 	@Test
-	public void testExpandByIdentifier() {
+	public void testExpandByIdentifier() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		Parameters respParam = ourClient
 			.operation()
 			.onType(ValueSet.class)
@@ -308,7 +423,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	}
 
 	@Test
-	public void testExpandByUrl() {
+	public void testExpandByUrl() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		Parameters respParam = ourClient
 			.operation()
 			.onType(ValueSet.class)
@@ -325,8 +442,70 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 	}
 
+	@Test
+	public void testExpandByUrlWithBogusUrl() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
+		try {
+			ourClient
+				.operation()
+				.onType(ValueSet.class)
+				.named("expand")
+				.withParameter(Parameters.class, "url", new UriType("http://www.healthintersections.com.au/fhir/ValueSet/bogus"))
+				.execute();
+		} catch (ResourceNotFoundException e) {
+			assertEquals(404, e.getStatusCode());
+			assertEquals("HTTP 404 Not Found: Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+		}
+	}
+
+	@Test
+	public void testExpandByUrlWithPreExpansion() throws Exception {
+		myDaoConfig.setPreExpandValueSetsExperimental(true);
+
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+
+		Parameters respParam = ourClient
+			.operation()
+			.onType(ValueSet.class)
+			.named("expand")
+			.withParameter(Parameters.class, "url", new UriType("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2"))
+			.execute();
+		ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
+
+		String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
+		ourLog.info(resp);
+		assertThat(resp, Matchers.stringContainsInOrder(
+			"<code value=\"11378-7\"/>",
+			"<display value=\"Systolic blood pressure at First encounter\"/>"));
+
+	}
+
+	@Test
+	public void testExpandByUrlWithPreExpansionAndBogusUrl() throws Exception {
+		myDaoConfig.setPreExpandValueSetsExperimental(true);
+
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+		myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
+
+		try {
+			Parameters respParam = ourClient
+				.operation()
+				.onType(ValueSet.class)
+				.named("expand")
+				.withParameter(Parameters.class, "url", new UriType("http://www.healthintersections.com.au/fhir/ValueSet/bogus"))
+				.execute();
+		} catch (ResourceNotFoundException e) {
+			assertEquals(404, e.getStatusCode());
+			assertEquals("HTTP 404 Not Found: Unknown ValueSet: http%3A%2F%2Fwww.healthintersections.com.au%2Ffhir%2FValueSet%2Fbogus", e.getMessage());
+		}
+	}
+
 	@Test
 	public void testExpandByValueSet() throws IOException {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		ValueSet toExpand = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
 
 		//@formatter:off
@@ -397,6 +576,8 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 	@Test
 	public void testExpandInvalidParams() throws IOException {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		try {
 			ourClient
 				.operation()
@@ -576,7 +757,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	}
 
 	@Test
-	public void testValidateCodeOperationByCodeAndSystemInstance() {
+	public void testValidateCodeOperationByCodeAndSystemInstance() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		//@formatter:off
 		Parameters respParam = ourClient
 			.operation()
@@ -595,6 +778,8 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 
 	@Test
 	public void testValidateCodeOperationByCodeAndSystemInstanceOnInstance() throws IOException {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		createLocalCsAndVs();
 		createLocalVsWithIncludeConcept();
 
@@ -617,7 +802,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	}
 
 	@Test
-	public void testValidateCodeOperationByCodeAndSystemType() {
+	public void testValidateCodeOperationByCodeAndSystemType() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		//@formatter:off
 		Parameters respParam = ourClient
 			.operation()
@@ -638,7 +825,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	 * Technically this is the wrong param name
 	 */
 	@Test
-	public void testValiedateCodeAgainstBuiltInSystem() {
+	public void testValiedateCodeAgainstBuiltInSystem() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		Parameters respParam = ourClient
 			.operation()
 			.onType(ValueSet.class)
@@ -666,7 +855,9 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 	 * Technically this is the right param name
 	 */
 	@Test
-	public void testValiedateCodeAgainstBuiltInSystemByUrl() {
+	public void testValiedateCodeAgainstBuiltInSystemByUrl() throws Exception {
+		loadAndPersistCodeSystemAndValueSet(HttpVerb.POST);
+
 		Parameters respParam = ourClient
 			.operation()
 			.onType(ValueSet.class)
@@ -690,6 +881,11 @@ public class ResourceProviderDstu3ValueSetTest extends BaseResourceProviderDstu3
 		assertEquals("Burn", ((StringType) respParam.getParameter().get(2).getValue()).getValue());
 	}
 
+	@After
+	public void afterResetPreExpansionDefault() {
+		myDaoConfig.setPreExpandValueSetsExperimental(new DaoConfig().isPreExpandValueSetsExperimental());
+	}
+
 	@AfterClass
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetTest.java

@@ -52,16 +52,6 @@ public class ResourceProviderR4ValueSetTest extends BaseResourceProviderR4Test {
 	private Long myExtensionalVsIdOnResourceTable;
 	private ValueSet myLocalVs;
 
-//	@Before
-//	@Transactional
-//	public void before02() throws IOException {
-//		CodeSystem cs = loadResourceFromClasspath(CodeSystem.class, "/extensional-case-3-cs.xml");
-//		myCodeSystemDao.create(cs, mySrd);
-//
-//		ValueSet upload = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml");
-//		myExtensionalVsId = myValueSetDao.create(upload, mySrd).getId().toUnqualifiedVersionless();
-//	}
-
 	private void loadAndPersistCodeSystemAndValueSet(HttpVerb theVerb) throws IOException {
 		loadAndPersistCodeSystem(theVerb);
 		loadAndPersistValueSet(theVerb);