From 6ec991cd39275ceba1cbbaae767ad7845ec26c0c Mon Sep 17 00:00:00 2001 From: James Agnew Date: Fri, 6 Sep 2019 16:29:09 -0400 Subject: [PATCH] Finally this is fast! --- .../dao/data/ITermValueSetConceptViewDao.java | 38 +++++ .../ca/uhn/fhir/jpa/entity/TermValueSet.java | 4 +- .../fhir/jpa/entity/TermValueSetConcept.java | 30 ++-- .../TermValueSetConceptDesignation.java | 32 +++-- .../jpa/entity/TermValueSetConceptView.java | 133 ++++++++++++++++++ .../jpa/term/BaseHapiTerminologySvcImpl.java | 76 ++++++---- 6 files changed, 255 insertions(+), 58 deletions(-) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java new file mode 100644 index 00000000000..323fd286228 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java @@ -0,0 +1,38 @@ +package ca.uhn.fhir.jpa.dao.data; + +import ca.uhn.fhir.jpa.entity.ResourceSearchView; +import ca.uhn.fhir.jpa.entity.TermValueSetConceptView; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.Collection; +import java.util.List; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +public interface ITermValueSetConceptViewDao extends JpaRepository { + + @Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptOrder >= :from AND v.myConceptOrder < :to ORDER BY v.myConceptOrder") + List findConceptsByValueSetPid(@Param("from") int theOffset, @Param("to") int theToIndex, @Param("pid") Long thePid); + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java index 3a694227906..8fd89ada6d5 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSet.java @@ -196,9 +196,7 @@ public class TermValueSet implements Serializable { @Override public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(getUrl()) - .toHashCode(); + return getUrl().hashCode(); } @Override diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java index 85ebf24a0ef..52bb91878de 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java @@ -53,7 +53,7 @@ public class TermValueSetConcept implements Serializable { @Column(name = "PID") private Long myId; - @ManyToOne() + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "VALUESET_PID", referencedColumnName = "PID", nullable = false, foreignKey = @ForeignKey(name = "FK_TRM_VALUESET_PID")) private TermValueSet myValueSet; @@ -78,9 +78,12 @@ public class TermValueSetConcept implements Serializable { @Column(name = "DISPLAY", nullable = true, length = TermConcept.MAX_DESC_LENGTH) private String myDisplay; - @OneToMany(mappedBy = "myConcept") + @OneToMany(mappedBy = "myConcept", fetch = FetchType.LAZY) private List myDesignations; + @Transient + private transient Integer myHashCode; + public Long getId() { return myId; } @@ -169,7 +172,7 @@ public class TermValueSetConcept implements Serializable { TermValueSetConcept that = (TermValueSetConcept) theO; return new EqualsBuilder() - .append(getValueSetUrl(), that.getValueSetUrl()) + .append(myValueSetPid, that.myValueSetPid) .append(getSystem(), that.getSystem()) .append(getCode(), that.getCode()) .isEquals(); @@ -177,26 +180,29 @@ public class TermValueSetConcept implements Serializable { @Override public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(getValueSetUrl()) - .append(getSystem()) - .append(getCode()) - .toHashCode(); + if (myHashCode == null) { + myHashCode = new HashCodeBuilder(17, 37) + .append(myValueSetPid) + .append(getSystem()) + .append(getCode()) + .toHashCode(); + } + return myHashCode; } @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE) .append("myId", myId) - .append(myValueSet != null ? ("myValueSet - id=" + myValueSet.getId()) : ("myValueSet=(null)")) +// .append(myValueSet != null ? ("myValueSet - id=" + myValueSet.getId()) : ("myValueSet=(null)")) .append("myValueSetPid", myValueSetPid) .append("myOrder", myOrder) - .append("myValueSetUrl", this.getValueSetUrl()) - .append("myValueSetName", this.getValueSetName()) +// .append("myValueSetUrl", this.getValueSetUrl()) +// .append("myValueSetName", this.getValueSetName()) .append("mySystem", mySystem) .append("myCode", myCode) .append("myDisplay", myDisplay) - .append(myDesignations != null ? ("myDesignations - size=" + myDesignations.size()) : ("myDesignations=(null)")) +// .append(myDesignations != null ? ("myDesignations - size=" + myDesignations.size()) : ("myDesignations=(null)")) .toString(); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptDesignation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptDesignation.java index 87e5ef2e8ec..2ec11b143b1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptDesignation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptDesignation.java @@ -48,14 +48,14 @@ public class TermValueSetConceptDesignation implements Serializable { @Column(name = "PID") private Long myId; - @ManyToOne() + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "VALUESET_CONCEPT_PID", referencedColumnName = "PID", nullable = false, foreignKey = @ForeignKey(name = "FK_TRM_VALUESET_CONCEPT_PID")) private TermValueSetConcept myConcept; @Column(name = "VALUESET_CONCEPT_PID", insertable = false, updatable = false, nullable = false) private Long myConceptPid; - @ManyToOne() + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "VALUESET_PID", referencedColumnName = "PID", nullable = false, foreignKey = @ForeignKey(name = "FK_TRM_VSCD_VS_PID")) private TermValueSet myValueSet; @@ -83,6 +83,9 @@ public class TermValueSetConceptDesignation implements Serializable { @Column(name = "VAL", nullable = false, length = MAX_LENGTH) private String myValue; + @Transient + private transient Integer myHashCode; + public Long getId() { return myId; } @@ -194,25 +197,28 @@ public class TermValueSetConceptDesignation implements Serializable { @Override public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(getLanguage()) - .append(getUseSystem()) - .append(getUseCode()) - .append(getUseDisplay()) - .append(getValue()) - .toHashCode(); + if (myHashCode == null) { + myHashCode = new HashCodeBuilder(17, 37) + .append(getLanguage()) + .append(getUseSystem()) + .append(getUseCode()) + .append(getUseDisplay()) + .append(getValue()) + .toHashCode(); + } + return myHashCode; } @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE) .append("myId", myId) - .append(myConcept != null ? ("myConcept - id=" + myConcept.getId()) : ("myConcept=(null)")) +// .append(myConcept != null ? ("myConcept - id=" + myConcept.getId()) : ("myConcept=(null)")) .append("myConceptPid", myConceptPid) - .append(myValueSet != null ? ("myValueSet - id=" + myValueSet.getId()) : ("myValueSet=(null)")) +// .append(myValueSet != null ? ("myValueSet - id=" + myValueSet.getId()) : ("myValueSet=(null)")) .append("myValueSetPid", myValueSetPid) - .append("myValueSetUrl", this.getValueSetUrl()) - .append("myValueSetName", this.getValueSetName()) +// .append("myValueSetUrl", this.getValueSetUrl()) +// .append("myValueSetName", this.getValueSetName()) .append("myLanguage", myLanguage) .append("myUseSystem", myUseSystem) .append("myUseCode", myUseCode) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java new file mode 100644 index 00000000000..5e9a13b23e5 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java @@ -0,0 +1,133 @@ +package ca.uhn.fhir.jpa.entity; + +/* + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.hibernate.annotations.Immutable; +import org.hibernate.annotations.Subselect; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import java.io.Serializable; + +@Entity +@Immutable +@Subselect("select " + + " CONCAT(vsc.PID, ' ', cd.PID) as PID," + + " vsc.PID as CONCEPT_PID, " + + " cd.PID as DESIG_PID, " + + " vsc.CODEVAL as CONCEPT_CODE, " + + " vsc.DISPLAY as CONCEPT_DISPLAY, " + + " vsc.VALUESET_ORDER as CONCEPT_ORDER, " + + " vsc.SYSTEM_URL as CONCEPT_SYSTEM_URL, " + + " vsc.VALUESET_PID as CONCEPT_VS_PID, " + +// " cd.VALUESET_CONCEPT_PID as VALUESET2_35_1_, " + + " cd.LANG as DESIG_LANG, " + + " cd.USE_CODE as DESIG_USE_CODE, " + + " cd.USE_DISPLAY as DESIG_USE_DISPLAY, " + + " cd.USE_SYSTEM as DESIG_USE_SYSTEM, " + + " cd.VAL as DESIG_VAL " + + " from TRM_VALUESET_CONCEPT vsc " + + " left outer join TRM_VALUESET_C_DESIGNATION cd on vsc.PID=cd.VALUESET_CONCEPT_PID " +// + +// " order by vsc.VALUESET_ORDER" +) +public class TermValueSetConceptView implements Serializable { + + private static final long serialVersionUID = 1L; + + @Id + @Column(name="PID") + private String myPid; + @Column(name = "CONCEPT_PID") + private Long myConceptPid; + @Column(name = "DESIG_PID") + private Long myDesigPid; + @Column(name = "CONCEPT_CODE") + private String myConceptCode; + @Column(name = "CONCEPT_DISPLAY") + private String myConceptDisplay; + @Column(name = "CONCEPT_ORDER") + private int myConceptOrder; + @Column(name = "CONCEPT_SYSTEM_URL") + private String myConceptSystemUrl; + @Column(name = "CONCEPT_VS_PID") + private Long myConceptValueSetPid; + @Column(name = "DESIG_LANG") + private String myDesigLang; + @Column(name = "DESIG_USE_CODE") + private String myDesigUseCode; + @Column(name = "DESIG_USE_DISPLAY") + private String myDesigUseDisplay; + @Column(name = "DESIG_USE_SYSTEM") + private String myDesigUseSystem; + @Column(name = "DESIG_VAL") + private String myDesigVal; + + + public Long getConceptPid() { + return myConceptPid; + } + + public Long getDesigPid() { + return myDesigPid; + } + + public String getConceptCode() { + return myConceptCode; + } + + public String getConceptDisplay() { + return myConceptDisplay; + } + + public int getConceptOrder() { + return myConceptOrder; + } + + public String getConceptSystemUrl() { + return myConceptSystemUrl; + } + + public Long getConceptValueSetPid() { + return myConceptValueSetPid; + } + + public String getDesigLang() { + return myDesigLang; + } + + public String getDesigUseCode() { + return myDesigUseCode; + } + + public String getDesigUseDisplay() { + return myDesigUseDisplay; + } + + public String getDesigUseSystem() { + return myDesigUseSystem; + } + + public String getDesigVal() { + return myDesigVal; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java index 9c8a9440436..a2f27e09200 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java @@ -68,6 +68,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.scheduling.annotation.Scheduled; @@ -159,6 +160,8 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, private IFulltextSearchSvc myFulltextSearchSvc; @Autowired private PlatformTransactionManager myTxManager; + @Autowired + private ITermValueSetConceptViewDao myTermValueSetConceptViewDao; private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set theAddedCodes, TermConcept theConcept, boolean theAdd, AtomicInteger theCodeCounter) { String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); @@ -528,21 +531,53 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, private void expandConcepts(ValueSet.ValueSetExpansionComponent theExpansionComponent, TermValueSet theTermValueSet, int theOffset, int theCount) { int conceptsExpanded = 0; int toIndex = theOffset + theCount; - Slice slice = myValueSetConceptDao.findByTermValueSetIdAndPreFetchDesignations(SearchCoordinatorSvcImpl.toPage(theOffset, toIndex), theTermValueSet.getId()); - if (!slice.hasContent()) { + +// Slice slice = myValueSetConceptDao.findByTermValueSetIdAndPreFetchDesignations(SearchCoordinatorSvcImpl.toPage(theOffset, toIndex), theTermValueSet.getId()); + Pageable page = SearchCoordinatorSvcImpl.toPage(theOffset, toIndex); + Collection slice = myTermValueSetConceptViewDao.findConceptsByValueSetPid(theOffset, toIndex, theTermValueSet.getId()); + + if (slice.isEmpty()) { logConceptsExpanded(theTermValueSet, conceptsExpanded); return; } - for (TermValueSetConcept concept : slice.getContent()) { - ValueSet.ValueSetExpansionContainsComponent containsComponent = theExpansionComponent.addContains(); - containsComponent.setSystem(concept.getSystem()); - containsComponent.setCode(concept.getCode()); - containsComponent.setDisplay(concept.getDisplay()); + Map pidToConcept = new HashMap<>(); + + int designationsExpanded = 0; + for (TermValueSetConceptView concept : slice) { + + Long conceptPid = concept.getConceptPid(); + ValueSet.ValueSetExpansionContainsComponent containsComponent; + + if (!pidToConcept.containsKey(conceptPid)) { + containsComponent = theExpansionComponent.addContains(); + containsComponent.setSystem(concept.getConceptSystemUrl()); + containsComponent.setCode(concept.getConceptCode()); + containsComponent.setDisplay(concept.getConceptDisplay()); + pidToConcept.put(conceptPid, containsComponent); + } else { + containsComponent = pidToConcept.get(conceptPid); + } + + if (concept.getDesigPid() != null) { + ValueSet.ConceptReferenceDesignationComponent designationComponent = containsComponent.addDesignation(); + designationComponent.setLanguage(concept.getDesigLang()); + designationComponent.setUse(new Coding( + concept.getDesigUseSystem(), + concept.getDesigUseCode(), + concept.getDesigUseDisplay())); + designationComponent.setValue(concept.getDesigVal()); + + if (++designationsExpanded % 250 == 0) { + logDesignationsExpanded(theTermValueSet, designationsExpanded); + } + + logDesignationsExpanded(theTermValueSet, designationsExpanded); + } // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to make this optional. // FIXME: DM 2019-09-05 - Let's try removing the pre-fetch and the code in BaseHapiTerminologySvcImpl that handles designations so we can compare the processing time. 2/2 - expandDesignations(theTermValueSet, concept, containsComponent); +// expandDesignations(theTermValueSet, concept, containsComponent); if (++conceptsExpanded % 250 == 0) { logConceptsExpanded(theTermValueSet, conceptsExpanded); @@ -559,30 +594,11 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, } } - private void expandDesignations(TermValueSet theValueSet, TermValueSetConcept theConcept, ValueSet.ValueSetExpansionContainsComponent theContainsComponent) { - int designationsExpanded = 0; - for (TermValueSetConceptDesignation designation : theConcept.getDesignations()) { - ValueSet.ConceptReferenceDesignationComponent designationComponent = theContainsComponent.addDesignation(); - designationComponent.setLanguage(designation.getLanguage()); - designationComponent.setUse(new Coding( - designation.getUseSystem(), - designation.getUseCode(), - designation.getUseDisplay())); - designationComponent.setValue(designation.getValue()); - - if (++designationsExpanded % 250 == 0) { - logDesignationsExpanded(theValueSet, theConcept, designationsExpanded); - } - } - - logDesignationsExpanded(theValueSet, theConcept, designationsExpanded); - } - - private void logDesignationsExpanded(TermValueSet theValueSet, TermValueSetConcept theConcept, int theDesignationsExpanded) { + private void logDesignationsExpanded(TermValueSet theValueSet, int theDesignationsExpanded) { if (theDesignationsExpanded > 0) { // FIXME: DM 2019-09-05 - Account for in progress vs. total. // FIXME: DM 2019-09-06 - Change to debug. - ourLog.info("Have expanded {} designations for Concept[{}|{}] in ValueSet[{}]", theDesignationsExpanded, theConcept.getSystem(), theConcept.getCode(), theValueSet.getUrl()); + ourLog.info("Have expanded {} designations in ValueSet[{}]", theDesignationsExpanded, theValueSet.getUrl()); } } @@ -972,7 +988,7 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, if (theCoding.hasSystem() && theCoding.hasCode()) { concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theCoding.getSystem(), theCoding.getCode())); } - } else if (theCodeableConcept != null){ + } else if (theCodeableConcept != null) { for (Coding coding : theCodeableConcept.getCoding()) { if (coding.hasSystem() && coding.hasCode()) { concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, coding.getSystem(), coding.getCode()));