Incremental work on large ValueSet expansion support; implemented paging when reading concepts from the terminology tables. Switched from Derby to H2.

This commit is contained in:
Diederik Muylwyk 2019-08-20 18:11:53 -04:00
parent 70aef7fbbb
commit 3218bd7853
8 changed files with 154 additions and 70 deletions

View File

@ -90,28 +90,12 @@
<artifactId>commons-cli</artifactId>
</dependency>
<!-- This example uses Derby embedded database. If you are using another database such as Mysql or Oracle, you may omit the following dependencies and replace them with an appropriate database client
<!-- This example uses H2 embedded database. If you are using another database such as Mysql or Oracle, you may omit the following dependencies and replace them with an appropriate database client
dependency for your database platform. -->
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derbynet</artifactId>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derbyclient</artifactId>
</dependency>
<!--<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derbyshared</artifactId>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derbytools</artifactId>
</dependency>-->
<!-- The following dependencies are only needed for automated unit tests, you do not neccesarily need them to run the example. -->
<dependency>

View File

@ -1,17 +1,16 @@
package ca.uhn.fhir.jpa.demo;
import java.util.Properties;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.model.entity.ModelConfig;
import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory;
import ca.uhn.fhir.jpa.util.DerbyTenSevenHapiFhirDialect;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.dialect.H2Dialect;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.sql.DataSource;
import java.util.Properties;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
@ -38,20 +37,20 @@ public class CommonConfig {
}
/**
* The following bean configures the database connection. The 'url' property value of "jdbc:derby:directory:jpaserver_derby_files;create=true" indicates that the server should save resources in a
* directory called "jpaserver_derby_files".
* The following bean configures the database connection. The 'url' property value of "jdbc:h2:file:target./jpaserver_h2_files" indicates that the server should save resources in a
* directory called "jpaserver_h2_files".
*
* A URL to a remote database could also be placed here, along with login credentials and other properties supported by BasicDataSource.
*/
@Bean(destroyMethod = "close")
public DataSource dataSource() {
String url = "jdbc:derby:directory:target/jpaserver_derby_files;create=true";
String url = "jdbc:h2:file:./target/jpaserver_h2_files";
if (isNotBlank(ContextHolder.getDatabaseUrl())) {
url = ContextHolder.getDatabaseUrl();
}
BasicDataSource retVal = new BasicDataSource();
retVal.setDriver(new org.apache.derby.jdbc.EmbeddedDriver());
retVal.setDriver(new org.h2.Driver());
retVal.setUrl(url);
retVal.setUsername("");
retVal.setPassword("");
@ -61,7 +60,7 @@ public class CommonConfig {
@Bean
public Properties jpaProperties() {
Properties extraProperties = new Properties();
extraProperties.put("hibernate.dialect", DerbyTenSevenHapiFhirDialect.class.getName());
extraProperties.put("hibernate.dialect", H2Dialect.class.getName());
extraProperties.put("hibernate.format_sql", "true");
extraProperties.put("hibernate.show_sql", "false");
extraProperties.put("hibernate.hbm2ddl.auto", "update");

View File

@ -21,6 +21,8 @@ package ca.uhn.fhir.jpa.dao.data;
*/
import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
@ -30,11 +32,17 @@ import java.util.Optional;
public interface ITermValueSetConceptDao extends JpaRepository<TermValueSetConcept, Long> {
@Query("SELECT COUNT(vsc) FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid")
Integer countByTermValueSetId(@Param("pid") Long theValueSetId);
@Query("DELETE FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid")
@Modifying
void deleteByTermValueSetId(@Param("pid") Long theValueSetId);
@Query("SELECT vsc from TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid")
Slice<TermValueSetConcept> findByTermValueSetId(Pageable thePage, @Param("pid") Long theValueSetId);
@Query("SELECT vsc FROM TermValueSetConcept vsc WHERE vsc.myValueSet.myId = :pid AND vsc.mySystem = :system_url AND vsc.myCode = :codeval")
Optional<TermValueSetConcept> findByValueSetIdSystemAndCode(@Param("pid") Long theValueSetId, @Param("system_url") String theSystem, @Param("codeval") String theCode);
Optional<TermValueSetConcept> findByTermValueSetIdSystemAndCode(@Param("pid") Long theValueSetId, @Param("system_url") String theSystem, @Param("codeval") String theCode);
}

View File

@ -21,6 +21,8 @@ package ca.uhn.fhir.jpa.dao.data;
*/
import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
@ -28,8 +30,13 @@ import org.springframework.data.repository.query.Param;
public interface ITermValueSetConceptDesignationDao extends JpaRepository<TermValueSetConceptDesignation, Long> {
@Query("SELECT COUNT(vscd) FROM TermValueSetConceptDesignation vscd WHERE vscd.myConcept.myId = :pid")
Integer countByTermValueSetConceptId(@Param("pid") Long theValueSetConceptId);
@Query("DELETE FROM TermValueSetConceptDesignation vscd WHERE vscd.myConcept.myValueSet.myId = :pid")
@Modifying
void deleteByTermValueSetId(@Param("pid") Long theValueSetId);
@Query("SELECT vscd from TermValueSetConceptDesignation vscd WHERE vscd.myConcept.myId = :pid")
Slice<TermValueSetConceptDesignation> findByTermValueSetConceptId(Pageable thePage, @Param("pid") Long theValueSetConceptId);
}

View File

@ -22,8 +22,8 @@ package ca.uhn.fhir.jpa.dao.data;
import ca.uhn.fhir.jpa.entity.TermValueSet;
import ca.uhn.fhir.jpa.entity.TermValueSetExpansionStatusEnum;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
@ -44,6 +44,6 @@ public interface ITermValueSetDao extends JpaRepository<TermValueSet, Long> {
Optional<TermValueSet> findByUrl(@Param("url") String theUrl);
@Query("SELECT vs FROM TermValueSet vs WHERE vs.myExpansionStatus = :expansion_status")
Page<TermValueSet> findByExpansionStatus(Pageable pageable, @Param("expansion_status") TermValueSetExpansionStatusEnum theExpansionStatus);
Slice<TermValueSet> findByExpansionStatus(Pageable pageable, @Param("expansion_status") TermValueSetExpansionStatusEnum theExpansionStatus);
}

View File

@ -420,7 +420,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
count = 0;
while (true) {
Slice<T> link = theLoader.get();
if (link.hasContent() == false) {
if (!link.hasContent()) {
break;
}
@ -484,7 +484,6 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
Optional<TermValueSet> optionalTermValueSet;
// FIXME: DM 2019-08-19 - This is no good. I need pageable results while querying TRM_VALUESET_CONCEPT
if (theValueSetToExpand.hasId()) {
optionalTermValueSet = myValueSetDao.findByResourcePid(theValueSetToExpand.getIdElement().getIdPartAsLong());
} else if (theValueSetToExpand.hasUrl()) {
@ -507,35 +506,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
expansionComponent.setIdentifier(UUID.randomUUID().toString());
expansionComponent.setTimestamp(new Date());
int numberOfConcepts = termValueSet.getConcepts().size();
expansionComponent.setTotal(numberOfConcepts);
expansionComponent.setOffset(theOffset);
List<TermValueSetConcept> subListOfConcepts = new ArrayList<>();
if (theCount != 0 && numberOfConcepts != 0) {
int toIndex = Math.min(theOffset + theCount, numberOfConcepts);
subListOfConcepts = termValueSet.getConcepts().subList(theOffset, toIndex);
}
for (TermValueSetConcept concept : subListOfConcepts) {
ValueSet.ValueSetExpansionContainsComponent containsComponent = expansionComponent.addContains();
containsComponent.setSystem(concept.getSystem());
containsComponent.setCode(concept.getCode());
containsComponent.setDisplay(concept.getDisplay());
// TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation.
for (TermValueSetConceptDesignation designation : concept.getDesignations()) {
ValueSet.ConceptReferenceDesignationComponent designationComponent = containsComponent.addDesignation();
designationComponent.setLanguage(designation.getLanguage());
if (isNoneBlank(designation.getUseSystem(), designation.getUseCode())) {
designationComponent.setUse(new Coding(
designation.getUseSystem(),
designation.getUseCode(),
designation.getUseDisplay()));
}
designationComponent.setValue(designation.getValue());
}
}
populateExpansionComponent(expansionComponent, termValueSet, theOffset, theCount);
ValueSet valueSet = new ValueSet();
valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
@ -544,6 +515,91 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
return valueSet;
}
private void populateExpansionComponent(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
int total = myValueSetConceptDao.countByTermValueSetId(theTermValueSet.getId());
theExpansionComponent.setTotal(total);
theExpansionComponent.setOffset(theOffset);
theExpansionComponent.addParameter().setName("offset").setValue(new IntegerType(theOffset));
theExpansionComponent.addParameter().setName("count").setValue(new IntegerType(theCount));
if (theCount == 0 || total == 0) {
return;
}
expandConcepts(theExpansionComponent, theTermValueSet, theOffset, theCount);
}
private void expandConcepts(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) {
int conceptsExpanded = 0;
for (int i = theOffset; i < (theOffset + theCount); i++) {
final int page = i;
Supplier<Slice<TermValueSetConcept>> loader = () -> myValueSetConceptDao.findByTermValueSetId(PageRequest.of(page, 1), theTermValueSet.getId());
Slice<TermValueSetConcept> slice = loader.get();
if (!slice.hasContent()) {
break;
}
for (TermValueSetConcept concept : slice.getContent()) {
ValueSet.ValueSetExpansionContainsComponent containsComponent = theExpansionComponent.addContains();
containsComponent.setSystem(concept.getSystem());
containsComponent.setCode(concept.getCode());
containsComponent.setDisplay(concept.getDisplay());
// TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to make this optional.
expandDesignations(theTermValueSet, concept, containsComponent);
if (++conceptsExpanded % 250 == 0) {
ourLog.info("Have expanded {} concepts in ValueSet[{}]", conceptsExpanded, theTermValueSet.getUrl());
}
}
if (!slice.hasNext()) {
break;
}
}
if (conceptsExpanded > 0) {
ourLog.info("Have expanded {} concepts in ValueSet[{}]", conceptsExpanded, theTermValueSet.getUrl());
}
}
private void expandDesignations(TermValueSet theValueSet, TermValueSetConcept theConcept, ValueSet.ValueSetExpansionContainsComponent theContainsComponent) {
int designationsExpanded = 0;
int index = 0;
while (true) {
final int page = index++;
Supplier<Slice<TermValueSetConceptDesignation>> loader = () -> myValueSetConceptDesignationDao.findByTermValueSetConceptId(PageRequest.of(page, 1000), theConcept.getId());
Slice<TermValueSetConceptDesignation> slice = loader.get();
if (!slice.hasContent()) {
break;
}
for (TermValueSetConceptDesignation designation : slice.getContent()) {
ValueSet.ConceptReferenceDesignationComponent designationComponent = theContainsComponent.addDesignation();
designationComponent.setLanguage(designation.getLanguage());
designationComponent.setUse(new Coding(
designation.getUseSystem(),
designation.getUseCode(),
designation.getUseDisplay()));
designationComponent.setValue(designation.getValue());
if (++designationsExpanded % 250 == 0) {
ourLog.info("Have expanded {} designations for Concept[{}|{}] in ValueSet[{}]", designationsExpanded, theConcept.getSystem(), theConcept.getCode(), theValueSet.getUrl());
}
}
if (!slice.hasNext()) {
break;
}
}
if (designationsExpanded > 0) {
ourLog.info("Have expanded {} designations for Concept[{}|{}] in ValueSet[{}]", designationsExpanded, theConcept.getSystem(), theConcept.getCode(), theValueSet.getUrl());
}
}
@Override
@Transactional(propagation = Propagation.REQUIRED)
public void expandValueSet(ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) {
@ -658,10 +714,10 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
String value = nextFilter.getValue();
if (value.endsWith("$")) {
value = value.substring(0, value.length() - 1);
} else if (value.endsWith(".*") == false) {
} else if (!value.endsWith(".*")) {
value = value + ".*";
}
if (value.startsWith("^") == false && value.startsWith(".*") == false) {
if (!value.startsWith("^") && !value.startsWith(".*")) {
value = ".*" + value;
} else if (value.startsWith("^")) {
value = value.substring(1);
@ -732,7 +788,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
throw new InvalidRequestException("Unknown code system: " + system);
}
if (theInclude.getConcept().isEmpty() == false) {
if (!theInclude.getConcept().isEmpty()) {
for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) {
String nextCode = next.getCode();
if (isNoneBlank(system, nextCode) && !theAddedCodes.contains(system + "|" + nextCode)) {
@ -857,7 +913,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
StopWatch stopwatch = new StopWatch();
Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
if (concept.isPresent() == false) {
if (!concept.isPresent()) {
return Collections.emptySet();
}
@ -888,7 +944,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
Stopwatch stopwatch = Stopwatch.createStarted();
Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
if (concept.isPresent() == false) {
if (!concept.isPresent()) {
return Collections.emptySet();
}
@ -1073,8 +1129,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
int maxResult = 1000;
Page<TermConcept> concepts = myConceptDao.findResourcesRequiringReindexing(new PageRequest(0, maxResult));
if (concepts.hasContent() == false) {
Page<TermConcept> concepts = myConceptDao.findResourcesRequiringReindexing(PageRequest.of(0, maxResult));
if (!concepts.hasContent()) {
if (myChildToParentPidCache != null) {
ourLog.info("Clearing parent concept cache");
myNextReindexPass = System.currentTimeMillis() + DateUtils.MILLIS_PER_MINUTE;
@ -1587,7 +1643,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
private Optional<TermValueSet> getNextTermValueSetNotExpanded() {
Optional<TermValueSet> retVal = Optional.empty();
Page<TermValueSet> page = myValueSetDao.findByExpansionStatus(PageRequest.of(0, 1), TermValueSetExpansionStatusEnum.NOT_EXPANDED);
Slice<TermValueSet> page = myValueSetDao.findByExpansionStatus(PageRequest.of(0, 1), TermValueSetExpansionStatusEnum.NOT_EXPANDED);
if (!page.getContent().isEmpty()) {
retVal = Optional.of(page.getContent().get(0));

View File

@ -71,7 +71,7 @@ public class ValueSetConceptAccumulator implements IValueSetConceptAccumulator {
}
// Get existing entity so it can be deleted.
Optional<TermValueSetConcept> optionalConcept = myValueSetConceptDao.findByValueSetIdSystemAndCode(myTermValueSet.getId(), theSystem, theCode);
Optional<TermValueSetConcept> optionalConcept = myValueSetConceptDao.findByTermValueSetIdSystemAndCode(myTermValueSet.getId(), theSystem, theCode);
if (optionalConcept.isPresent()) {
TermValueSetConcept concept = optionalConcept.get();

View File

@ -614,6 +614,12 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffsetExperimental(), expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(1000, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getContains().size());
ValueSet.ValueSetExpansionContainsComponent containsComponent = expandedValueSet.getExpansion().getContains().get(0);
@ -683,6 +689,12 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffsetExperimental(), expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(23, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertEquals(23, expandedValueSet.getExpansion().getContains().size());
ValueSet.ValueSetExpansionContainsComponent containsComponent = expandedValueSet.getExpansion().getContains().get(0);
@ -746,6 +758,12 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(myDaoConfig.getPreExpandValueSetsDefaultOffsetExperimental(), expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(0, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(0, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertFalse(expandedValueSet.getExpansion().hasContains());
}
@ -768,6 +786,12 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(1, expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(1, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(1000, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertEquals(codeSystem.getConcept().size() - expandedValueSet.getExpansion().getOffset(), expandedValueSet.getExpansion().getContains().size());
ValueSet.ValueSetExpansionContainsComponent containsComponent = expandedValueSet.getExpansion().getContains().get(0);
@ -823,6 +847,12 @@ public class TerminologySvcImplR4Test extends BaseJpaR4Test {
assertEquals(codeSystem.getConcept().size(), expandedValueSet.getExpansion().getTotal());
assertEquals(1, expandedValueSet.getExpansion().getOffset());
assertEquals(2, expandedValueSet.getExpansion().getParameter().size());
assertEquals("offset", expandedValueSet.getExpansion().getParameter().get(0).getName());
assertEquals(1, expandedValueSet.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue());
assertEquals("count", expandedValueSet.getExpansion().getParameter().get(1).getName());
assertEquals(22, expandedValueSet.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue());
assertEquals(22, expandedValueSet.getExpansion().getContains().size());
ValueSet.ValueSetExpansionContainsComponent containsComponent = expandedValueSet.getExpansion().getContains().get(0);