From 116cb1c8f76e609c3167ed2fd13626b5ae8a73e3 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 6 Feb 2017 21:57:23 -0500 Subject: [PATCH] Work on repopulating --- .../java/ca/uhn/fhir/util/ParametersUtil.java | 2 + .../uhn/fhir/jpa/config/BaseDstu2Config.java | 1 + .../data/ITermConceptParentChildLinkDao.java | 5 + .../ca/uhn/fhir/jpa/entity/TermConcept.java | 18 ++- .../entity/TermConceptParentChildLink.java | 130 ++++++++++-------- .../fhir/jpa/term/BaseHapiTerminologySvc.java | 66 ++++++++- .../jpa/term/HapiTerminologySvcDstu3.java | 65 +++++---- .../FhirResourceDaoDstu3TerminologyTest.java | 71 ++++++++++ .../jpa/provider/SystemProviderDstu2Test.java | 40 ++++-- .../dstu3/SystemProviderDstu3Test.java | 34 ++--- 10 files changed, 302 insertions(+), 130 deletions(-) diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ParametersUtil.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ParametersUtil.java index 339dbe18c83..68e7ae16614 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ParametersUtil.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ParametersUtil.java @@ -22,6 +22,7 @@ package ca.uhn.fhir.util; import java.util.Collection; +import org.apache.commons.lang3.Validate; import org.hl7.fhir.instance.model.api.IBase; import org.hl7.fhir.instance.model.api.IBaseDatatype; import org.hl7.fhir.instance.model.api.IBaseParameters; @@ -84,6 +85,7 @@ public class ParametersUtil { } public static IBaseParameters newInstance(FhirContext theContext) { + Validate.notNull(theContext, "theContext must not be null"); return (IBaseParameters) theContext.getResourceDefinition("Parameters").newInstance(); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseDstu2Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseDstu2Config.java index 32a5021cd45..20f694e62b1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseDstu2Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseDstu2Config.java @@ -101,6 +101,7 @@ public class BaseDstu2Config extends BaseConfig { public ca.uhn.fhir.jpa.provider.JpaSystemProviderDstu2 systemProviderDstu2() { ca.uhn.fhir.jpa.provider.JpaSystemProviderDstu2 retVal = new ca.uhn.fhir.jpa.provider.JpaSystemProviderDstu2(); retVal.setDao(systemDaoDstu2()); + retVal.setContext(fhirContextDstu2()); return retVal; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java index b663163edcd..d4d315a3c4f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java @@ -1,5 +1,7 @@ package ca.uhn.fhir.jpa.dao.data; +import java.util.Collection; + /* * #%L * HAPI FHIR JPA Server @@ -32,5 +34,8 @@ public interface ITermConceptParentChildLinkDao extends JpaRepository findAllWithChild(@Param("child_pid") Long theConceptPid); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java index 4c5182d0ffa..5ae32fdd272 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java @@ -233,12 +233,14 @@ public class TermConcept implements Serializable { @PreUpdate @PrePersist public void prePersist() { - Set parentPids = new HashSet(); - TermConcept entity = this; - parentPids(entity, parentPids); - entity.setParentPids(parentPids); - - ourLog.trace("Code {}/{} has parents {}", entity.getId(), entity.getCode(), entity.getParentPidsAsString()); + if (myParentPids == null) { + Set parentPids = new HashSet(); + TermConcept entity = this; + parentPids(entity, parentPids); + entity.setParentPids(parentPids); + + ourLog.trace("Code {}/{} has parents {}", entity.getId(), entity.getCode(), entity.getParentPidsAsString()); + } } public void setCode(String theCode) { @@ -280,6 +282,10 @@ public class TermConcept implements Serializable { myParentPids = b.toString(); } + public void setParentPids(String theParentPids) { + myParentPids = theParentPids; + } + @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("code", myCode).append("display", myDisplay).build(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java index 09887b012a3..f7ddb31c580 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java @@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.entity; * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -36,77 +36,38 @@ import javax.persistence.SequenceGenerator; import javax.persistence.Table; @Entity -@Table(name="TRM_CONCEPT_PC_LINK") +@Table(name = "TRM_CONCEPT_PC_LINK") public class TermConceptParentChildLink implements Serializable { private static final long serialVersionUID = 1L; @ManyToOne() - @JoinColumn(name="CHILD_PID", nullable=false, referencedColumnName="PID", foreignKey=@ForeignKey(name="FK_TERM_CONCEPTPC_CHILD")) + @JoinColumn(name = "CHILD_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_CHILD")) private TermConcept myChild; + @Column(name = "CHILD_PID", insertable = false, updatable = false) + private Long myChildPid; + @ManyToOne() - @JoinColumn(name="CODESYSTEM_PID", nullable=false, foreignKey=@ForeignKey(name="FK_TERM_CONCEPTPC_CS")) + @JoinColumn(name = "CODESYSTEM_PID", nullable = false, foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_CS")) private TermCodeSystemVersion myCodeSystem; - @ManyToOne(cascade= {}) - @JoinColumn(name="PARENT_PID", nullable=false, referencedColumnName="PID", foreignKey=@ForeignKey(name="FK_TERM_CONCEPTPC_PARENT")) + @ManyToOne(cascade = {}) + @JoinColumn(name = "PARENT_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_PARENT")) private TermConcept myParent; + @Column(name = "PARENT_PID", insertable = false, updatable = false) + private Long myParentPid; + @Id() - @SequenceGenerator(name="SEQ_CONCEPT_PC_PID", sequenceName="SEQ_CONCEPT_PC_PID") - @GeneratedValue(strategy=GenerationType.AUTO, generator="SEQ_CONCEPT_PC_PID") - @Column(name="PID") + @SequenceGenerator(name = "SEQ_CONCEPT_PC_PID", sequenceName = "SEQ_CONCEPT_PC_PID") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CONCEPT_PC_PID") + @Column(name = "PID") private Long myPid; @Enumerated(EnumType.ORDINAL) - @Column(name="REL_TYPE", length=5, nullable=true) + @Column(name = "REL_TYPE", length = 5, nullable = true) private RelationshipTypeEnum myRelationshipType; - public TermConcept getChild() { - return myChild; - } - - public RelationshipTypeEnum getRelationshipType() { - return myRelationshipType; - } - - public TermCodeSystemVersion getCodeSystem() { - return myCodeSystem; - } - - public TermConcept getParent() { - return myParent; - } - - public void setChild(TermConcept theChild) { - myChild = theChild; - } - - public void setCodeSystem(TermCodeSystemVersion theCodeSystem) { - myCodeSystem = theCodeSystem; - } - - public void setParent(TermConcept theParent) { - myParent = theParent; - } - - - public void setRelationshipType(RelationshipTypeEnum theRelationshipType) { - myRelationshipType = theRelationshipType; - } - - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((myChild == null) ? 0 : myChild.hashCode()); - result = prime * result + ((myCodeSystem == null) ? 0 : myCodeSystem.hashCode()); - result = prime * result + ((myParent == null) ? 0 : myParent.hashCode()); - result = prime * result + ((myRelationshipType == null) ? 0 : myRelationshipType.hashCode()); - return result; - } - @Override public boolean equals(Object obj) { if (this == obj) @@ -136,15 +97,64 @@ public class TermConceptParentChildLink implements Serializable { return true; } - - public enum RelationshipTypeEnum{ - // ******************************************** - // IF YOU ADD HERE MAKE SURE ORDER IS PRESERVED - ISA + public TermConcept getChild() { + return myChild; } + public Long getChildPid() { + return myChildPid; + } + + public TermCodeSystemVersion getCodeSystem() { + return myCodeSystem; + } public Long getId() { return myPid; } + + public TermConcept getParent() { + return myParent; + } + + public Long getParentPid() { + return myParentPid; + } + + public RelationshipTypeEnum getRelationshipType() { + return myRelationshipType; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((myChild == null) ? 0 : myChild.hashCode()); + result = prime * result + ((myCodeSystem == null) ? 0 : myCodeSystem.hashCode()); + result = prime * result + ((myParent == null) ? 0 : myParent.hashCode()); + result = prime * result + ((myRelationshipType == null) ? 0 : myRelationshipType.hashCode()); + return result; + } + + public void setChild(TermConcept theChild) { + myChild = theChild; + } + + public void setCodeSystem(TermCodeSystemVersion theCodeSystem) { + myCodeSystem = theCodeSystem; + } + + public void setParent(TermConcept theParent) { + myParent = theParent; + } + + public void setRelationshipType(RelationshipTypeEnum theRelationshipType) { + myRelationshipType = theRelationshipType; + } + + public enum RelationshipTypeEnum { + // ******************************************** + // IF YOU ADD HERE MAKE SURE ORDER IS PRESERVED + ISA + } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvc.java index f8bbb377f5d..765e09cf340 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvc.java @@ -23,9 +23,11 @@ package ca.uhn.fhir.jpa.term; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -33,6 +35,7 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.PersistenceContextType; +import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.time.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; @@ -47,6 +50,8 @@ import org.springframework.transaction.support.TransactionTemplate; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimaps; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; @@ -305,6 +310,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc { } } + private ArrayListMultimap myChildToParentPidCache; + private void processReindexing() { if (System.currentTimeMillis() < myNextReindexPass && !ourForceSaveDeferredAlwaysForUnitTest) { return; @@ -316,23 +323,60 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc { @Override protected void doInTransactionWithoutResult(TransactionStatus theArg0) { int maxResult = 1000; - Page resources = myConceptDao.findResourcesRequiringReindexing(new PageRequest(0, maxResult)); - if (resources.hasContent() == false) { + Page concepts = myConceptDao.findResourcesRequiringReindexing(new PageRequest(0, maxResult)); + if (concepts.hasContent() == false) { myNextReindexPass = System.currentTimeMillis() + DateUtils.MILLIS_PER_MINUTE; + myChildToParentPidCache = null; return; } - ourLog.info("Indexing {} / {} concepts", resources.getContent().size(), resources.getTotalElements()); + if (myChildToParentPidCache == null) { + myChildToParentPidCache = ArrayListMultimap.create(); + } + + ourLog.info("Indexing {} / {} concepts", concepts.getContent().size(), concepts.getTotalElements()); int count = 0; StopWatch stopwatch = new StopWatch(); - for (TermConcept resourceTable : resources) { - saveConcept(resourceTable); + for (TermConcept nextConcept : concepts) { + + StringBuilder parentsBuilder = new StringBuilder(); + createParentsString(parentsBuilder, nextConcept.getId()); + nextConcept.setParentPids(parentsBuilder.toString()); + + saveConcept(nextConcept); count++; } - ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, resources.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) }); + ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) }); + } + + private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) { + Validate.notNull(theConceptPid, "theConceptPid must not be null"); + List parents = myChildToParentPidCache.get(theConceptPid); + if (parents.contains(-1L)) { + return; + } else if (parents.isEmpty()) { + Collection parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid); + if (parentLinks.isEmpty()) { + myChildToParentPidCache.put(theConceptPid, -1L); + return; + } else { + for (TermConceptParentChildLink next : parentLinks) { + myChildToParentPidCache.put(theConceptPid, next.getParentPid()); + } + } + } + + + for (Long nextParent : parents) { + if (theParentsBuilder.length() > 0) { + theParentsBuilder.append(' '); + } + theParentsBuilder.append(nextParent); + createParentsString(theParentsBuilder, nextParent); + } } }); @@ -340,7 +384,15 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc { private int saveConcept(TermConcept theConcept) { int retVal = 0; - retVal += ensureParentsSaved(theConcept.getParents()); + + /* + * If the concept has an ID, we're reindexing, so there's no need to + * save parent concepts first (it's way too slow to do that) + */ + if (theConcept.getId() == null) { + retVal += ensureParentsSaved(theConcept.getParents()); + } + if (theConcept.getId() == null || theConcept.getIndexStatus() == null) { retVal++; theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java index add64802014..96a2191bfa6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.term; +import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; /* @@ -12,7 +13,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -64,6 +65,7 @@ import ca.uhn.fhir.jpa.entity.ResourceTable; import ca.uhn.fhir.jpa.entity.TermCodeSystem; import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.util.StopWatch; import ca.uhn.fhir.rest.method.RequestDetails; import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; @@ -79,7 +81,7 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I @Autowired private IValidationSupport myValidationSupport; - + private void addCodeIfNotAlreadyAdded(String system, ValueSetExpansionComponent retVal, Set addedCodes, TermConcept nextConcept) { if (addedCodes.add(nextConcept.getCode())) { ValueSetExpansionContainsComponent contains = retVal.addContains(); @@ -126,12 +128,12 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I for (ConceptDefinitionComponent nextChild : theNext.getConcept()) { foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate); } - + if (theCode.equals(theNext.getCode()) || foundCodeInChild) { theListToPopulate.add(new VersionIndependentConcept(theSystemString, theNext.getCode())); return true; } - + return false; } @@ -159,17 +161,15 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I } private void addDisplayFilterInexact(QueryBuilder qb, BooleanJunction bool, ConceptSetFilterComponent nextFilter) { - //@formatter:off Query textQuery = qb .phrase() .withSlop(2) .onField("myDisplay").boostedTo(4.0f) .andField("myDisplayEdgeNGram").boostedTo(2.0f) - //.andField("myDisplayNGram").boostedTo(1.0f) - //.andField("myDisplayPhonetic").boostedTo(0.5f) + // .andField("myDisplayNGram").boostedTo(1.0f) + // .andField("myDisplayPhonetic").boostedTo(0.5f) .sentence(nextFilter.getValue().toLowerCase()).createQuery(); bool.must(textQuery); - //@formatter:on } @Override @@ -216,26 +216,33 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I bool.must(qb.keyword().onField("myCodeSystemVersionPid").matching(csv.getPid()).createQuery()); for (ConceptSetFilterComponent nextFilter : theInclude.getFilter()) { - if (isNotBlank(nextFilter.getValue())) { - if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == FilterOperator.EQUAL) { - addDisplayFilterExact(qb, bool, nextFilter); - } else if (nextFilter.getProperty().equals("display") && nextFilter.getOp() == FilterOperator.EQUAL) { - if (nextFilter.getValue().trim().contains(" ")) { - addDisplayFilterExact(qb, bool, nextFilter); - } else { - addDisplayFilterInexact(qb, bool, nextFilter); - } - } else if ((nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) && nextFilter.getOp() == FilterOperator.ISA) { - TermConcept code = super.findCode(system, nextFilter.getValue()); - if (code == null) { - throw new InvalidRequestException("Invalid filter criteria - code does not exist: {" + system + "}" + nextFilter.getValue()); - } + if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) { + continue; + } - ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay()); - bool.must(qb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery()); + if (isBlank(nextFilter.getValue()) || nextFilter.getOp() == null || isBlank(nextFilter.getProperty())) { + throw new InvalidRequestException("Invalid filter, must have fields populated: property op value"); + } + + + if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == FilterOperator.EQUAL) { + addDisplayFilterExact(qb, bool, nextFilter); + } else if ("display".equals(nextFilter.getProperty()) && nextFilter.getOp() == FilterOperator.EQUAL) { + if (nextFilter.getValue().trim().contains(" ")) { + addDisplayFilterExact(qb, bool, nextFilter); } else { - throw new InvalidRequestException("Unknown filter property[" + nextFilter + "] + op[" + nextFilter.getOpElement().getValueAsString() + "]"); + addDisplayFilterInexact(qb, bool, nextFilter); } + } else if ((nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) && nextFilter.getOp() == FilterOperator.ISA) { + TermConcept code = super.findCode(system, nextFilter.getValue()); + if (code == null) { + throw new InvalidRequestException("Invalid filter criteria - code does not exist: {" + system + "}" + nextFilter.getValue()); + } + + ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay()); + bool.must(qb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery()); + } else { + throw new InvalidRequestException("Unknown filter property[" + nextFilter + "] + op[" + nextFilter.getOpElement().getValueAsString() + "]"); } } @@ -243,12 +250,17 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class); jpaQuery.setMaxResults(1000); + StopWatch sw = new StopWatch(); + @SuppressWarnings("unchecked") List result = jpaQuery.getResultList(); + + ourLog.info("Expansion completed in {}ms", sw.getMillis()); + for (TermConcept nextConcept : result) { addCodeIfNotAlreadyAdded(system, retVal, addedCodes, nextConcept); } - + retVal.setTotal(jpaQuery.getResultSize()); } @@ -259,7 +271,6 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I } } - return retVal; } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java index 2efb85d863e..414622d3a30 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java @@ -26,7 +26,9 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import ca.uhn.fhir.jpa.dao.BaseHapiFhirSystemDao; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult; import ca.uhn.fhir.jpa.dao.SearchParameterMap; @@ -34,6 +36,8 @@ import ca.uhn.fhir.jpa.entity.ResourceTable; import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; +import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvc; +import ca.uhn.fhir.jpa.term.IHapiTerminologySvc; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.param.TokenParamModifier; @@ -52,6 +56,8 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test { @After public void after() { myDaoConfig.setDeferIndexingForCodesystemsOfSize(new DaoConfig().getDeferIndexingForCodesystemsOfSize()); + + BaseHapiTerminologySvc.setForceSaveDeferredAlwaysForUnitTest(false); } @Before @@ -318,6 +324,71 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test { } } + @Test + public void testExpandWithIsAInExternalValueSet() { + createExternalCsAndLocalVs(); + + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + include.addFilter().setOp(FilterOperator.ISA).setValue("childAA").setProperty("concept"); + + ValueSet result = myValueSetDao.expand(vs, null); + logAndValidateValueSet(result); + + ArrayList codes = toCodesContains(result.getExpansion().getContains()); + assertThat(codes, containsInAnyOrder("childAAA", "childAAB")); + + } + + @Autowired + private IHapiTerminologySvc myHapiTerminologySvc; + + @Test + public void testExpandWithIsAInExternalValueSetReindex() { + BaseHapiTerminologySvc.setForceSaveDeferredAlwaysForUnitTest(true); + + createExternalCsAndLocalVs(); + + mySystemDao.markAllResourcesForReindexing(); + + mySystemDao.performReindexingPass(100); + mySystemDao.performReindexingPass(100); + myHapiTerminologySvc.saveDeferred(); + myHapiTerminologySvc.saveDeferred(); + myHapiTerminologySvc.saveDeferred(); + + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + include.addFilter().setOp(FilterOperator.ISA).setValue("childAA").setProperty("concept"); + + ValueSet result = myValueSetDao.expand(vs, null); + logAndValidateValueSet(result); + + ArrayList codes = toCodesContains(result.getExpansion().getContains()); + assertThat(codes, containsInAnyOrder("childAAA", "childAAB")); + + } + + @Test + public void testExpandInvalid() { + createExternalCsAndLocalVs(); + + ValueSet vs = new ValueSet(); + ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(URL_MY_CODE_SYSTEM); + include.addFilter(); + include.addFilter().setOp(FilterOperator.ISA).setValue("childAA"); + + try { + myValueSetDao.expand(vs, null); + fail(); + } catch (InvalidRequestException e) { + assertEquals("Invalid filter, must have fields populated: property op value", e.getMessage()); + } + } + @Test public void testExpandWithSystemAndCodesInExternalValueSet() { createExternalCsAndLocalVs(); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/SystemProviderDstu2Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/SystemProviderDstu2Test.java index 4fd7a995c19..3977ff8992e 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/SystemProviderDstu2Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/SystemProviderDstu2Test.java @@ -10,6 +10,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; @@ -139,7 +140,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { get.addHeader("Accept", "application/xml, text/html"); CloseableHttpResponse http = ourHttpClient.execute(get); try { - String response = IOUtils.toString(http.getEntity().getContent()); + String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(response); assertThat(response, (containsString("_format=json"))); assertEquals(200, http.getStatusLine().getStatusCode()); @@ -167,7 +168,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { get.addHeader("Accept", "application/xml+fhir"); CloseableHttpResponse http = ourHttpClient.execute(get); try { - String response = IOUtils.toString(http.getEntity().getContent()); + String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(response); assertThat(response, not(containsString("_format"))); assertEquals(200, http.getStatusLine().getStatusCode()); @@ -216,7 +217,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { CloseableHttpResponse http = ourHttpClient.execute(get); try { assertEquals(200, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); Parameters parameters = ourCtx.newXmlParser().parseResource(Parameters.class, output); @@ -246,7 +247,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { CloseableHttpResponse http = ourHttpClient.execute(get); try { assertEquals(400, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertThat(output, containsString("Parameter 'context' must be provided")); } finally { @@ -257,7 +258,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { http = ourHttpClient.execute(get); try { assertEquals(400, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertThat(output, containsString("Parameter 'searchParam' must be provided")); } finally { @@ -268,7 +269,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { http = ourHttpClient.execute(get); try { assertEquals(400, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertThat(output, containsString("Parameter 'text' must be provided")); } finally { @@ -286,7 +287,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { @Test public void testTransactionFromBundle() throws Exception { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/transaction_link_patient_eve.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); } @@ -295,7 +296,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { public void testTransactionFromBundle2() throws Exception { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); @@ -311,7 +312,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { */ bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml"); - bundle = IOUtils.toString(bundleRes); + bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); @@ -335,7 +336,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { public void testTransactionFromBundle3() throws Exception { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/grahame-transaction.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); } @@ -343,7 +344,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { @Test public void testTransactionFromBundle4() throws Exception { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/simone_bundle.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); Bundle bundleResp = ourCtx.newXmlParser().parseResource(Bundle.class, response); @@ -358,7 +359,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { @Test public void testTransactionFromBundle5() throws Exception { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/simone_bundle2.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); try { ourClient.transaction().withBundle(bundle).prettyPrint().execute(); fail(); @@ -372,7 +373,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { @Test public void testTransactionFromBundle6() throws Exception { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/simone_bundle3.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); ourClient.transaction().withBundle(bundle).prettyPrint().execute(); // try { // fail(); @@ -427,5 +428,18 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test { assertEquals(0, respSub.getEntry().size()); } + @Test + public void testMarkResourcesForReindexing() throws Exception { + HttpGet get = new HttpGet(ourServerBase + "/$mark-all-resources-for-reindexing"); + CloseableHttpResponse http = ourHttpClient.execute(get); + try { + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); + ourLog.info(output); + assertEquals(200, http.getStatusLine().getStatusCode()); + } finally { + IOUtils.closeQuietly(http);; + } + } + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/SystemProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/SystemProviderDstu3Test.java index c78abae2d65..e4c70769d94 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/SystemProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/SystemProviderDstu3Test.java @@ -129,7 +129,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { CloseableHttpResponse resp = ourHttpClient.execute(req); try { - String encoded = IOUtils.toString(resp.getEntity().getContent()); + String encoded = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(encoded); assertThat(encoded, containsString("transaction-response")); @@ -162,7 +162,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { CloseableHttpResponse resp = ourHttpClient.execute(req); try { - String encoded = IOUtils.toString(resp.getEntity().getContent()); + String encoded = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(encoded); assertThat(encoded, containsString("transaction-response")); @@ -271,14 +271,14 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { try { InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/questionnaire-sdc-profile-example-ussg-fht.xml"); - String bundleStr = IOUtils.toString(bundleRes); + String bundleStr = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); HttpPost req = new HttpPost(ourServerBase); req.setEntity(new StringEntity(bundleStr, ContentType.parse(Constants.CT_FHIR_XML + "; charset=utf-8"))); CloseableHttpResponse resp = ourHttpClient.execute(req); try { - String encoded = IOUtils.toString(resp.getEntity().getContent()); + String encoded = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(encoded); //@formatter:off @@ -314,7 +314,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { CloseableHttpResponse http = ourHttpClient.execute(get); try { - String response = IOUtils.toString(http.getEntity().getContent()); + String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(response); assertThat(response, containsString("_format=json")); assertEquals(200, http.getStatusLine().getStatusCode()); @@ -342,7 +342,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { get.addHeader("Accept", "application/xml+fhir"); CloseableHttpResponse http = ourHttpClient.execute(get); try { - String response = IOUtils.toString(http.getEntity().getContent()); + String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(response); assertThat(response, not(containsString("_format"))); assertEquals(200, http.getStatusLine().getStatusCode()); @@ -373,7 +373,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { HttpGet get = new HttpGet(ourServerBase + "/$mark-all-resources-for-reindexing"); CloseableHttpResponse http = ourHttpClient.execute(get); try { - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertEquals(200, http.getStatusLine().getStatusCode()); } finally { @@ -404,7 +404,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { CloseableHttpResponse http = ourHttpClient.execute(get); try { assertEquals(200, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); Parameters parameters = ourCtx.newXmlParser().parseResource(Parameters.class, output); @@ -434,7 +434,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { CloseableHttpResponse http = ourHttpClient.execute(get); try { assertEquals(400, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertThat(output, containsString("Parameter 'context' must be provided")); } finally { @@ -445,7 +445,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { http = ourHttpClient.execute(get); try { assertEquals(400, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertThat(output, containsString("Parameter 'searchParam' must be provided")); } finally { @@ -456,7 +456,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { http = ourHttpClient.execute(get); try { assertEquals(400, http.getStatusLine().getStatusCode()); - String output = IOUtils.toString(http.getEntity().getContent()); + String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(output); assertThat(output, containsString("Parameter 'text' must be provided")); } finally { @@ -500,7 +500,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { public void testTransactionFromBundle2() throws Exception { InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); @@ -516,7 +516,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { */ bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml"); - bundle = IOUtils.toString(bundleRes); + bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); @@ -540,7 +540,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { public void testTransactionFromBundle3() throws Exception { InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/grahame-transaction.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); } @@ -548,7 +548,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { @Test public void testTransactionFromBundle4() throws Exception { InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/simone_bundle.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute(); ourLog.info(response); Bundle bundleResp = ourCtx.newXmlParser().parseResource(Bundle.class, response); @@ -563,7 +563,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { @Test public void testTransactionFromBundle5() throws Exception { InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/simone_bundle2.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); try { ourClient.transaction().withBundle(bundle).prettyPrint().execute(); fail(); @@ -577,7 +577,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test { @Test public void testTransactionFromBundle6() throws Exception { InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/simone_bundle3.xml"); - String bundle = IOUtils.toString(bundleRes); + String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8); ourClient.transaction().withBundle(bundle).prettyPrint().execute(); // try { // fail();