Merge pull request #1958 from jamesagnew/im_20200630_code_system_load

Im 20200630 code system load

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoObservationDstu3.java

@@ -21,7 +21,6 @@ package ca.uhn.fhir.jpa.dao.dstu3;
  */
 
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation;
-import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc;
 import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
@@ -32,16 +31,12 @@ import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.dstu3.model.Observation;
-import org.springframework.beans.factory.annotation.Autowired;
 
 import javax.servlet.http.HttpServletResponse;
 import java.util.Date;
 
 public class FhirResourceDaoObservationDstu3 extends BaseHapiFhirResourceDaoObservation<Observation> {
 
-	@Autowired
-	ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc;
-
 	@Override
 	public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap,  RequestDetails theRequestDetails, HttpServletResponse theServletResponse) {
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoObservationR4.java

@@ -21,7 +21,6 @@ package ca.uhn.fhir.jpa.dao.r4;
  */
 
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation;
-import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc;
 import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
@@ -32,7 +31,6 @@ import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.Observation;
-import org.springframework.beans.factory.annotation.Autowired;
 
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
@@ -46,9 +44,6 @@ public class FhirResourceDaoObservationR4 extends BaseHapiFhirResourceDaoObserva
 	@PersistenceContext(type = PersistenceContextType.TRANSACTION)
 	protected EntityManager myEntityManager;
 
-	@Autowired
-	ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc;
-
 	@Override
 	public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap,  RequestDetails theRequestDetails, HttpServletResponse theServletResponse) {
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r5/FhirResourceDaoObservationR5.java

@@ -21,7 +21,6 @@ package ca.uhn.fhir.jpa.dao.r5;
  */
 
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDaoObservation;
-import ca.uhn.fhir.jpa.dao.ObservationLastNIndexPersistSvc;
 import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
@@ -32,16 +31,12 @@ import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r5.model.Observation;
-import org.springframework.beans.factory.annotation.Autowired;
 
 import javax.servlet.http.HttpServletResponse;
 import java.util.Date;
 
 public class FhirResourceDaoObservationR5 extends BaseHapiFhirResourceDaoObservation<Observation> {
 
-	@Autowired
-	ObservationLastNIndexPersistSvc myObservationLastNIndexPersistSvc;
-
 	@Override
 	public IBundleProvider observationsLastN(SearchParameterMap theSearchParameterMap,  RequestDetails theRequestDetails, HttpServletResponse theServletResponse) {
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java

@@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.term;
 
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
 import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
+import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
 import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
 import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
 import ca.uhn.fhir.jpa.entity.TermCodeSystem;
@@ -62,6 +63,8 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 	@Autowired
 	protected ITermCodeSystemDao myCodeSystemDao;
 	@Autowired
+   protected ITermCodeSystemVersionDao myCodeSystemVersionDao;
+   @Autowired
 	protected PlatformTransactionManager myTransactionMgr;
 	private boolean myProcessDeferred = true;
 	private List<TermCodeSystem> myDefferedCodeSystemsDeletions = Collections.synchronizedList(new ArrayList<>());
@@ -142,7 +145,12 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 		ourLog.info("Saving {} deferred concepts...", count);
 		while (codeCount < count && myDeferredConcepts.size() > 0) {
 			TermConcept next = myDeferredConcepts.remove(0);
-			codeCount += myCodeSystemStorageSvc.saveConcept(next);
+			if(myCodeSystemVersionDao.findById(next.getCodeSystemVersion().getPid()).isPresent()) {
+         	codeCount += myCodeSystemStorageSvc.saveConcept(next);
+			} else {
+				ourLog.warn("Unable to save deferred TermConcept {} because Code System {} version PID {} is no longer valid. Code system may have since been replaced.",
+					next.getCode(), next.getCodeSystemVersion().getCodeSystemDisplayName(), next.getCodeSystemVersion().getPid());
+			}
 		}
 
 		if (codeCount > 0) {
@@ -340,4 +348,9 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
 	void setConceptDaoForUnitTest(ITermConceptDao theConceptDao) {
 		myConceptDao = theConceptDao;
 	}
+
+	@VisibleForTesting
+	void setCodeSystemVersionDaoForUnitTest(ITermCodeSystemVersionDao theCodeSystemVersionDao) {
+		myCodeSystemVersionDao = theCodeSystemVersionDao;
+	}
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/PersistObservationIndexedSearchParamLastNR4IT.java

@@ -203,9 +203,6 @@ public class PersistObservationIndexedSearchParamLastNR4IT {
 		List<ObservationJson> observationDocuments = elasticsearchSvc.executeLastNWithAllFieldsForTest(searchParameterMap, myFhirCtx);
 		assertEquals(100, observationDocuments.size());
 
-		//List<CodeJson> codeDocuments = elasticsearchSvc.queryAllIndexedObservationCodesForTest();
-		//assertEquals(2, codeDocuments.size());
-
 		// Check that all observations were indexed.
 		searchParameterMap = new SearchParameterMap();
 		searchParameterMap.add(Observation.SP_SUBJECT, multiSubjectParams);

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcTest.java

@@ -0,0 +1,44 @@
+package ca.uhn.fhir.jpa.term;
+
+import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import org.hl7.fhir.r4.model.CodeSystem;
+import org.hl7.fhir.r4.model.CodeType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TermCodeSystemStorageSvcTest extends BaseJpaR4Test {
+
+	public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system";
+
+	private CodeSystem createCodeSystemWithMoreThan100Concepts() {
+		CodeSystem codeSystem = new CodeSystem();
+		codeSystem.setUrl(URL_MY_CODE_SYSTEM);
+		codeSystem.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
+
+		for (int i = 0; i < 125; i++) {
+			codeSystem.addConcept(new CodeSystem.ConceptDefinitionComponent(new CodeType("codeA " + i)));
+		}
+
+		return codeSystem;
+
+	}
+
+	@Test
+	public void testStoreNewCodeSystemVersionForExistingCodeSystem() {
+		CodeSystem upload = createCodeSystemWithMoreThan100Concepts();
+
+		ResourceTable codeSystemResourceEntity = (ResourceTable)myCodeSystemDao.create(upload, mySrd).getEntity();
+
+		runInTransaction(() -> myTermCodeSystemStorageSvc.storeNewCodeSystemVersionIfNeeded(upload, codeSystemResourceEntity));
+
+		myTerminologyDeferredStorageSvc.setProcessDeferred(true);
+		myTerminologyDeferredStorageSvc.saveDeferred();
+		myTerminologyDeferredStorageSvc.saveDeferred();
+
+		assertEquals(125, myTermConceptDao.count());
+	}
+
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImplTest.java

@@ -1,7 +1,9 @@
 package ca.uhn.fhir.jpa.term;
 
 import ca.uhn.fhir.jpa.api.config.DaoConfig;
+import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
 import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
+import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
 import ca.uhn.fhir.jpa.entity.TermConcept;
 import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
 import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
@@ -11,6 +13,8 @@ import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.transaction.PlatformTransactionManager;
 
+import java.util.Optional;
+
 import static org.mockito.ArgumentMatchers.same;
 import static org.mockito.Mockito.*;
 
@@ -24,6 +28,8 @@ public class TermDeferredStorageSvcImplTest {
 	private ITermCodeSystemStorageSvc myTermConceptStorageSvc;
 	@Mock
 	private ITermConceptDao myConceptDao;
+	@Mock
+	private ITermCodeSystemVersionDao myTermCodeSystemVersionDao;
 
 	@Test
 	public void testSaveDeferredWithExecutionSuspended() {
@@ -38,16 +44,45 @@ public class TermDeferredStorageSvcImplTest {
 		TermConcept concept = new TermConcept();
 		concept.setCode("CODE_A");
 
+		TermCodeSystemVersion myTermCodeSystemVersion = new TermCodeSystemVersion();
+		myTermCodeSystemVersion.setId(1L);
+		concept.setCodeSystemVersion(myTermCodeSystemVersion);
+
 		TermDeferredStorageSvcImpl svc = new TermDeferredStorageSvcImpl();
 		svc.setTransactionManagerForUnitTest(myTxManager);
 		svc.setCodeSystemStorageSvcForUnitTest(myTermConceptStorageSvc);
 		svc.setDaoConfigForUnitTest(new DaoConfig());
+
+		when(myTermCodeSystemVersionDao.findById(anyLong())).thenReturn(Optional.of(myTermCodeSystemVersion));
+		svc.setCodeSystemVersionDaoForUnitTest(myTermCodeSystemVersionDao);
 		svc.setProcessDeferred(true);
 		svc.addConceptToStorageQueue(concept);
 		svc.saveDeferred();
-
 		verify(myTermConceptStorageSvc, times(1)).saveConcept(same(concept));
 		verifyNoMoreInteractions(myTermConceptStorageSvc);
+
+	}
+
+	@Test
+	public void testSaveDeferred_Concept_StaleCodeSystemVersion() {
+		TermConcept concept = new TermConcept();
+		concept.setCode("CODE_A");
+
+		TermCodeSystemVersion myTermCodeSystemVersion = new TermCodeSystemVersion();
+		concept.setCodeSystemVersion(myTermCodeSystemVersion);
+
+		TermDeferredStorageSvcImpl svc = new TermDeferredStorageSvcImpl();
+		svc.setTransactionManagerForUnitTest(myTxManager);
+		svc.setCodeSystemStorageSvcForUnitTest(myTermConceptStorageSvc);
+		svc.setDaoConfigForUnitTest(new DaoConfig());
+
+		svc.setCodeSystemVersionDaoForUnitTest(myTermCodeSystemVersionDao);
+		svc.setProcessDeferred(true);
+		svc.addConceptToStorageQueue(concept);
+		svc.saveDeferred();
+		verify(myTermConceptStorageSvc, times(0)).saveConcept(same(concept));
+		verifyNoMoreInteractions(myTermConceptStorageSvc);
+
 	}
 
 	@Test