Work on repopulating

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ParametersUtil.java

@@ -22,6 +22,7 @@ package ca.uhn.fhir.util;
 
 import java.util.Collection;
 
+import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseDatatype;
 import org.hl7.fhir.instance.model.api.IBaseParameters;
@@ -84,6 +85,7 @@ public class ParametersUtil {
 	}
 
 	public static IBaseParameters newInstance(FhirContext theContext) {
+		Validate.notNull(theContext, "theContext must not be null");
 		return (IBaseParameters) theContext.getResourceDefinition("Parameters").newInstance();
 	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseDstu2Config.java

@@ -101,6 +101,7 @@ public class BaseDstu2Config extends BaseConfig {
 	public ca.uhn.fhir.jpa.provider.JpaSystemProviderDstu2 systemProviderDstu2() {
 		ca.uhn.fhir.jpa.provider.JpaSystemProviderDstu2 retVal = new ca.uhn.fhir.jpa.provider.JpaSystemProviderDstu2();
 		retVal.setDao(systemDaoDstu2());
+		retVal.setContext(fhirContextDstu2());
 		return retVal;
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermConceptParentChildLinkDao.java

@@ -1,5 +1,7 @@
 package ca.uhn.fhir.jpa.dao.data;
 
+import java.util.Collection;
+
 /*
  * #%L
  * HAPI FHIR JPA Server
@@ -32,5 +34,8 @@ public interface ITermConceptParentChildLinkDao extends JpaRepository<TermConcep
 	@Query("DELETE FROM TermConceptParentChildLink t WHERE t.myCodeSystem.myId = :cs_pid")
 	@Modifying
 	void deleteByCodeSystemVersion(@Param("cs_pid") Long thePid);
+
+	@Query("SELECT t FROM TermConceptParentChildLink t WHERE t.myChildPid = :child_pid")
+	Collection<TermConceptParentChildLink> findAllWithChild(@Param("child_pid") Long theConceptPid);
 	
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java

@@ -233,12 +233,14 @@ public class TermConcept implements Serializable {
 	@PreUpdate
 	@PrePersist
 	public void prePersist() {
-		Set<Long> parentPids = new HashSet<Long>();
+		if (myParentPids == null) {
-		TermConcept entity = this;
+			Set<Long> parentPids = new HashSet<Long>();
-		parentPids(entity, parentPids);
+			TermConcept entity = this;
-		entity.setParentPids(parentPids);
+			parentPids(entity, parentPids);
-
+			entity.setParentPids(parentPids);
-		ourLog.trace("Code {}/{} has parents {}", entity.getId(), entity.getCode(), entity.getParentPidsAsString());
+	
+			ourLog.trace("Code {}/{} has parents {}", entity.getId(), entity.getCode(), entity.getParentPidsAsString());
+		}
 	}
 
 	public void setCode(String theCode) {
@@ -280,6 +282,10 @@ public class TermConcept implements Serializable {
 		myParentPids = b.toString();
 	}
 
+	public void setParentPids(String theParentPids) {
+		myParentPids = theParentPids;
+	}
+
 	@Override
 	public String toString() {
 		return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("code", myCode).append("display", myDisplay).build();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptParentChildLink.java

@@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.entity;
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  * 
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  * 
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -36,77 +36,38 @@ import javax.persistence.SequenceGenerator;
 import javax.persistence.Table;
 
 @Entity
-@Table(name="TRM_CONCEPT_PC_LINK")
+@Table(name = "TRM_CONCEPT_PC_LINK")
 public class TermConceptParentChildLink implements Serializable {
 	private static final long serialVersionUID = 1L;
 
 	@ManyToOne()
-	@JoinColumn(name="CHILD_PID", nullable=false, referencedColumnName="PID", foreignKey=@ForeignKey(name="FK_TERM_CONCEPTPC_CHILD"))
+	@JoinColumn(name = "CHILD_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_CHILD"))
 	private TermConcept myChild;
 
+	@Column(name = "CHILD_PID", insertable = false, updatable = false)
+	private Long myChildPid;
+
 	@ManyToOne()
-	@JoinColumn(name="CODESYSTEM_PID", nullable=false, foreignKey=@ForeignKey(name="FK_TERM_CONCEPTPC_CS"))
+	@JoinColumn(name = "CODESYSTEM_PID", nullable = false, foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_CS"))
 	private TermCodeSystemVersion myCodeSystem;
 
-	@ManyToOne(cascade= {})
+	@ManyToOne(cascade = {})
-	@JoinColumn(name="PARENT_PID", nullable=false, referencedColumnName="PID", foreignKey=@ForeignKey(name="FK_TERM_CONCEPTPC_PARENT"))
+	@JoinColumn(name = "PARENT_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TERM_CONCEPTPC_PARENT"))
 	private TermConcept myParent;
 
+	@Column(name = "PARENT_PID", insertable = false, updatable = false)
+	private Long myParentPid;
+
 	@Id()
-	@SequenceGenerator(name="SEQ_CONCEPT_PC_PID", sequenceName="SEQ_CONCEPT_PC_PID")
+	@SequenceGenerator(name = "SEQ_CONCEPT_PC_PID", sequenceName = "SEQ_CONCEPT_PC_PID")
-	@GeneratedValue(strategy=GenerationType.AUTO, generator="SEQ_CONCEPT_PC_PID")
+	@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_CONCEPT_PC_PID")
-	@Column(name="PID")
+	@Column(name = "PID")
 	private Long myPid;
 
 	@Enumerated(EnumType.ORDINAL)
-	@Column(name="REL_TYPE", length=5, nullable=true)
+	@Column(name = "REL_TYPE", length = 5, nullable = true)
 	private RelationshipTypeEnum myRelationshipType;
 
-	public TermConcept getChild() {
-		return myChild;
-	}
-
-	public RelationshipTypeEnum getRelationshipType() {
-		return myRelationshipType;
-	}
-
-	public TermCodeSystemVersion getCodeSystem() {
-		return myCodeSystem;
-	}
-	
-	public TermConcept getParent() {
-		return myParent;
-	}
-
-	public void setChild(TermConcept theChild) {
-		myChild = theChild;
-	}
-	
-	public void setCodeSystem(TermCodeSystemVersion theCodeSystem) {
-		myCodeSystem = theCodeSystem;
-	}
-
-	public void setParent(TermConcept theParent) {
-		myParent = theParent;
-	}
-	
-	
-	public void setRelationshipType(RelationshipTypeEnum theRelationshipType) {
-		myRelationshipType = theRelationshipType;
-	}
-
-
-	@Override
-	public int hashCode() {
-		final int prime = 31;
-		int result = 1;
-		result = prime * result + ((myChild == null) ? 0 : myChild.hashCode());
-		result = prime * result + ((myCodeSystem == null) ? 0 : myCodeSystem.hashCode());
-		result = prime * result + ((myParent == null) ? 0 : myParent.hashCode());
-		result = prime * result + ((myRelationshipType == null) ? 0 : myRelationshipType.hashCode());
-		return result;
-	}
-
 	@Override
 	public boolean equals(Object obj) {
 		if (this == obj)
@@ -136,15 +97,64 @@ public class TermConceptParentChildLink implements Serializable {
 		return true;
 	}
 
-
+	public TermConcept getChild() {
-	public enum RelationshipTypeEnum{
+		return myChild;
-		// ********************************************
-		// IF YOU ADD HERE MAKE SURE ORDER IS PRESERVED
-		ISA
 	}
 
+	public Long getChildPid() {
+		return myChildPid;
+	}
+
+	public TermCodeSystemVersion getCodeSystem() {
+		return myCodeSystem;
+	}
 
 	public Long getId() {
 		return myPid;
 	}
+
+	public TermConcept getParent() {
+		return myParent;
+	}
+
+	public Long getParentPid() {
+		return myParentPid;
+	}
+
+	public RelationshipTypeEnum getRelationshipType() {
+		return myRelationshipType;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((myChild == null) ? 0 : myChild.hashCode());
+		result = prime * result + ((myCodeSystem == null) ? 0 : myCodeSystem.hashCode());
+		result = prime * result + ((myParent == null) ? 0 : myParent.hashCode());
+		result = prime * result + ((myRelationshipType == null) ? 0 : myRelationshipType.hashCode());
+		return result;
+	}
+
+	public void setChild(TermConcept theChild) {
+		myChild = theChild;
+	}
+
+	public void setCodeSystem(TermCodeSystemVersion theCodeSystem) {
+		myCodeSystem = theCodeSystem;
+	}
+
+	public void setParent(TermConcept theParent) {
+		myParent = theParent;
+	}
+
+	public void setRelationshipType(RelationshipTypeEnum theRelationshipType) {
+		myRelationshipType = theRelationshipType;
+	}
+
+	public enum RelationshipTypeEnum {
+		// ********************************************
+		// IF YOU ADD HERE MAKE SURE ORDER IS PRESERVED
+		ISA
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvc.java

@@ -23,9 +23,11 @@ package ca.uhn.fhir.jpa.term;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -33,6 +35,7 @@ import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
 import javax.persistence.PersistenceContextType;
 
+import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
@@ -47,6 +50,8 @@ import org.springframework.transaction.support.TransactionTemplate;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimaps;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
@@ -305,6 +310,8 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
 		}
 	}
 
+	private ArrayListMultimap<Long, Long> myChildToParentPidCache;
+	
 	private void processReindexing() {
 		if (System.currentTimeMillis() < myNextReindexPass && !ourForceSaveDeferredAlwaysForUnitTest) {
 			return;
@@ -316,23 +323,60 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
 			@Override
 			protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
 				int maxResult = 1000;
-				Page<TermConcept> resources = myConceptDao.findResourcesRequiringReindexing(new PageRequest(0, maxResult));
+				Page<TermConcept> concepts = myConceptDao.findResourcesRequiringReindexing(new PageRequest(0, maxResult));
-				if (resources.hasContent() == false) {
+				if (concepts.hasContent() == false) {
 					myNextReindexPass = System.currentTimeMillis() + DateUtils.MILLIS_PER_MINUTE;
+					myChildToParentPidCache = null;
 					return;
 				}
 
-				ourLog.info("Indexing {} / {} concepts", resources.getContent().size(), resources.getTotalElements());
+				if (myChildToParentPidCache == null) {
+					myChildToParentPidCache = ArrayListMultimap.create();
+				}
+				
+				ourLog.info("Indexing {} / {} concepts", concepts.getContent().size(), concepts.getTotalElements());
 
 				int count = 0;
 				StopWatch stopwatch = new StopWatch();
 
-				for (TermConcept resourceTable : resources) {
+				for (TermConcept nextConcept : concepts) {
-					saveConcept(resourceTable);
+					
+					StringBuilder parentsBuilder = new StringBuilder();
+					createParentsString(parentsBuilder, nextConcept.getId());
+					nextConcept.setParentPids(parentsBuilder.toString());
+					
+					saveConcept(nextConcept);
 					count++;
 				}
 				
-				ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, resources.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) });
+				ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", new Object[] { count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count) });
+			}
+
+			private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) {
+				Validate.notNull(theConceptPid, "theConceptPid must not be null");
+				List<Long> parents = myChildToParentPidCache.get(theConceptPid);
+				if (parents.contains(-1L)) {
+					return;
+				} else if (parents.isEmpty()) {
+					Collection<TermConceptParentChildLink> parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid);
+					if (parentLinks.isEmpty()) {
+						myChildToParentPidCache.put(theConceptPid, -1L);
+						return;
+					} else {
+						for (TermConceptParentChildLink next : parentLinks) {
+							myChildToParentPidCache.put(theConceptPid, next.getParentPid());
+						}
+					}
+				}
+				
+				
+				for (Long nextParent : parents) {
+					if (theParentsBuilder.length() > 0) {
+						theParentsBuilder.append(' ');
+					}
+					theParentsBuilder.append(nextParent);
+					createParentsString(theParentsBuilder, nextParent);
+				}
 			}
 		});
 
@@ -340,7 +384,15 @@ public abstract class BaseHapiTerminologySvc implements IHapiTerminologySvc {
 
 	private int saveConcept(TermConcept theConcept) {
 		int retVal = 0;
-		retVal += ensureParentsSaved(theConcept.getParents());
+	
+		/*
+		 * If the concept has an ID, we're reindexing, so there's no need to
+		 * save parent concepts first (it's way too slow to do that)
+		 */
+		if (theConcept.getId() == null) {
+			retVal += ensureParentsSaved(theConcept.getParents());
+		}
+		
 		if (theConcept.getId() == null || theConcept.getIndexStatus() == null) {
 			retVal++;
 			theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/HapiTerminologySvcDstu3.java

@@ -1,5 +1,6 @@
 package ca.uhn.fhir.jpa.term;
 
+import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 /*
@@ -12,7 +13,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  * 
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  * 
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -64,6 +65,7 @@ import ca.uhn.fhir.jpa.entity.ResourceTable;
 import ca.uhn.fhir.jpa.entity.TermCodeSystem;
 import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
 import ca.uhn.fhir.jpa.entity.TermConcept;
+import ca.uhn.fhir.jpa.util.StopWatch;
 import ca.uhn.fhir.rest.method.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
@@ -79,7 +81,7 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I
 
 	@Autowired
 	private IValidationSupport myValidationSupport;
-	
+
 	private void addCodeIfNotAlreadyAdded(String system, ValueSetExpansionComponent retVal, Set<String> addedCodes, TermConcept nextConcept) {
 		if (addedCodes.add(nextConcept.getCode())) {
 			ValueSetExpansionContainsComponent contains = retVal.addContains();
@@ -126,12 +128,12 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I
 		for (ConceptDefinitionComponent nextChild : theNext.getConcept()) {
 			foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate);
 		}
-		
+
 		if (theCode.equals(theNext.getCode()) || foundCodeInChild) {
 			theListToPopulate.add(new VersionIndependentConcept(theSystemString, theNext.getCode()));
 			return true;
 		}
-		
+
 		return false;
 	}
 
@@ -159,17 +161,15 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I
 	}
 
 	private void addDisplayFilterInexact(QueryBuilder qb, BooleanJunction<?> bool, ConceptSetFilterComponent nextFilter) {
-		//@formatter:off
 		Query textQuery = qb
 				.phrase()
 				.withSlop(2)
 				.onField("myDisplay").boostedTo(4.0f)
 				.andField("myDisplayEdgeNGram").boostedTo(2.0f)
-				//.andField("myDisplayNGram").boostedTo(1.0f)
+				// .andField("myDisplayNGram").boostedTo(1.0f)
-				//.andField("myDisplayPhonetic").boostedTo(0.5f)
+				// .andField("myDisplayPhonetic").boostedTo(0.5f)
 				.sentence(nextFilter.getValue().toLowerCase()).createQuery();
 		bool.must(textQuery);
-		//@formatter:on
 	}
 
 	@Override
@@ -216,26 +216,33 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I
 			bool.must(qb.keyword().onField("myCodeSystemVersionPid").matching(csv.getPid()).createQuery());
 
 			for (ConceptSetFilterComponent nextFilter : theInclude.getFilter()) {
-				if (isNotBlank(nextFilter.getValue())) {
+				if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) {
-					if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == FilterOperator.EQUAL) {
+					continue;
-						addDisplayFilterExact(qb, bool, nextFilter);
+				}
-					} else if (nextFilter.getProperty().equals("display") && nextFilter.getOp() == FilterOperator.EQUAL) {
-						if (nextFilter.getValue().trim().contains(" ")) {
-							addDisplayFilterExact(qb, bool, nextFilter);
-						} else {
-							addDisplayFilterInexact(qb, bool, nextFilter);
-						}
-					} else if ((nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) && nextFilter.getOp() == FilterOperator.ISA) {
-						TermConcept code = super.findCode(system, nextFilter.getValue());
-						if (code == null) {
-							throw new InvalidRequestException("Invalid filter criteria - code does not exist: {" + system + "}" + nextFilter.getValue());
-						}
 
-						ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
+				if (isBlank(nextFilter.getValue()) || nextFilter.getOp() == null || isBlank(nextFilter.getProperty())) {
-						bool.must(qb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery());
+					throw new InvalidRequestException("Invalid filter, must have fields populated: property op value");
+				}
+				
+				
+				if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == FilterOperator.EQUAL) {
+					addDisplayFilterExact(qb, bool, nextFilter);
+				} else if ("display".equals(nextFilter.getProperty()) && nextFilter.getOp() == FilterOperator.EQUAL) {
+					if (nextFilter.getValue().trim().contains(" ")) {
+						addDisplayFilterExact(qb, bool, nextFilter);
 					} else {
-						throw new InvalidRequestException("Unknown filter property[" + nextFilter + "] + op[" + nextFilter.getOpElement().getValueAsString() + "]");
+						addDisplayFilterInexact(qb, bool, nextFilter);
 					}
+				} else if ((nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) && nextFilter.getOp() == FilterOperator.ISA) {
+					TermConcept code = super.findCode(system, nextFilter.getValue());
+					if (code == null) {
+						throw new InvalidRequestException("Invalid filter criteria - code does not exist: {" + system + "}" + nextFilter.getValue());
+					}
+
+					ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
+					bool.must(qb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery());
+				} else {
+					throw new InvalidRequestException("Unknown filter property[" + nextFilter + "] + op[" + nextFilter.getOpElement().getValueAsString() + "]");
 				}
 			}
 
@@ -243,12 +250,17 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I
 			FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class);
 			jpaQuery.setMaxResults(1000);
 
+			StopWatch sw = new StopWatch();
+			
 			@SuppressWarnings("unchecked")
 			List<TermConcept> result = jpaQuery.getResultList();
+			
+			ourLog.info("Expansion completed in {}ms", sw.getMillis());
+			
 			for (TermConcept nextConcept : result) {
 				addCodeIfNotAlreadyAdded(system, retVal, addedCodes, nextConcept);
 			}
-			
+
 			retVal.setTotal(jpaQuery.getResultSize());
 		}
 
@@ -259,7 +271,6 @@ public class HapiTerminologySvcDstu3 extends BaseHapiTerminologySvc implements I
 			}
 		}
 
-
 		return retVal;
 	}
 

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java

@@ -26,7 +26,9 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 
+import ca.uhn.fhir.jpa.dao.BaseHapiFhirSystemDao;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem.LookupCodeResult;
 import ca.uhn.fhir.jpa.dao.SearchParameterMap;
@@ -34,6 +36,8 @@ import ca.uhn.fhir.jpa.entity.ResourceTable;
 import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
 import ca.uhn.fhir.jpa.entity.TermConcept;
 import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
+import ca.uhn.fhir.jpa.term.BaseHapiTerminologySvc;
+import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
 import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.TokenParamModifier;
@@ -52,6 +56,8 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
 	@After
 	public void after() {
 		myDaoConfig.setDeferIndexingForCodesystemsOfSize(new DaoConfig().getDeferIndexingForCodesystemsOfSize());
+		
+		BaseHapiTerminologySvc.setForceSaveDeferredAlwaysForUnitTest(false);
 	}
 
 	@Before
@@ -318,6 +324,71 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
 		}
 	}
 
+	@Test
+	public void testExpandWithIsAInExternalValueSet() {
+		createExternalCsAndLocalVs();
+
+		ValueSet vs = new ValueSet();
+		ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem(URL_MY_CODE_SYSTEM);
+		include.addFilter().setOp(FilterOperator.ISA).setValue("childAA").setProperty("concept");
+
+		ValueSet result = myValueSetDao.expand(vs, null);
+		logAndValidateValueSet(result);
+
+		ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
+		assertThat(codes, containsInAnyOrder("childAAA", "childAAB"));
+
+	}
+
+	@Autowired
+	private IHapiTerminologySvc myHapiTerminologySvc;
+	
+	@Test
+	public void testExpandWithIsAInExternalValueSetReindex() {
+		BaseHapiTerminologySvc.setForceSaveDeferredAlwaysForUnitTest(true);
+		
+		createExternalCsAndLocalVs();
+
+		mySystemDao.markAllResourcesForReindexing();
+
+		mySystemDao.performReindexingPass(100);
+		mySystemDao.performReindexingPass(100);
+		myHapiTerminologySvc.saveDeferred();
+		myHapiTerminologySvc.saveDeferred();
+		myHapiTerminologySvc.saveDeferred();
+		
+		ValueSet vs = new ValueSet();
+		ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem(URL_MY_CODE_SYSTEM);
+		include.addFilter().setOp(FilterOperator.ISA).setValue("childAA").setProperty("concept");
+
+		ValueSet result = myValueSetDao.expand(vs, null);
+		logAndValidateValueSet(result);
+
+		ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
+		assertThat(codes, containsInAnyOrder("childAAA", "childAAB"));
+
+	}
+
+	@Test
+	public void testExpandInvalid() {
+		createExternalCsAndLocalVs();
+
+		ValueSet vs = new ValueSet();
+		ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem(URL_MY_CODE_SYSTEM);
+		include.addFilter();
+		include.addFilter().setOp(FilterOperator.ISA).setValue("childAA");
+
+		try {
+			myValueSetDao.expand(vs, null);
+			fail();
+		} catch (InvalidRequestException e) {
+			assertEquals("Invalid filter, must have fields populated: property op value", e.getMessage());
+		}
+	}
+
 	@Test
 	public void testExpandWithSystemAndCodesInExternalValueSet() {
 		createExternalCsAndLocalVs();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/SystemProviderDstu2Test.java

@@ -10,6 +10,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.IOUtils;
@@ -139,7 +140,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		get.addHeader("Accept", "application/xml, text/html");
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
-			String response = IOUtils.toString(http.getEntity().getContent());
+			String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(response);
 			assertThat(response, (containsString("_format=json")));
 			assertEquals(200, http.getStatusLine().getStatusCode());
@@ -167,7 +168,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		get.addHeader("Accept", "application/xml+fhir");
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
-			String response = IOUtils.toString(http.getEntity().getContent());
+			String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(response);
 			assertThat(response, not(containsString("_format")));
 			assertEquals(200, http.getStatusLine().getStatusCode());
@@ -216,7 +217,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
 			assertEquals(200, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			
 			Parameters parameters = ourCtx.newXmlParser().parseResource(Parameters.class, output);
@@ -246,7 +247,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
 			assertEquals(400, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertThat(output, containsString("Parameter 'context' must be provided"));
 		} finally {
@@ -257,7 +258,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		http = ourHttpClient.execute(get);
 		try {
 			assertEquals(400, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertThat(output, containsString("Parameter 'searchParam' must be provided"));
 		} finally {
@@ -268,7 +269,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		http = ourHttpClient.execute(get);
 		try {
 			assertEquals(400, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertThat(output, containsString("Parameter 'text' must be provided"));
 		} finally {
@@ -286,7 +287,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 	@Test
 	public void testTransactionFromBundle() throws Exception {
 		InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/transaction_link_patient_eve.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 	}
@@ -295,7 +296,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 	public void testTransactionFromBundle2() throws Exception {
 
 		InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 
@@ -311,7 +312,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		 */
 
 		bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml");
-		bundle = IOUtils.toString(bundleRes);
+		bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 
@@ -335,7 +336,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 	public void testTransactionFromBundle3() throws Exception {
 
 		InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/grahame-transaction.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 	}
@@ -343,7 +344,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 	@Test
 	public void testTransactionFromBundle4() throws Exception {
 		InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/simone_bundle.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 		Bundle bundleResp = ourCtx.newXmlParser().parseResource(Bundle.class, response);
@@ -358,7 +359,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 	@Test
 	public void testTransactionFromBundle5() throws Exception {
 		InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/simone_bundle2.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		try {
 			ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 			fail();
@@ -372,7 +373,7 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 	@Test
 	public void testTransactionFromBundle6() throws Exception {
 		InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/simone_bundle3.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		// try {
 		// fail();
@@ -427,5 +428,18 @@ public class SystemProviderDstu2Test extends BaseJpaDstu2Test {
 		assertEquals(0, respSub.getEntry().size());
 	}
 
+	@Test
+	public void testMarkResourcesForReindexing() throws Exception {
+		HttpGet get = new HttpGet(ourServerBase + "/$mark-all-resources-for-reindexing");
+		CloseableHttpResponse http = ourHttpClient.execute(get);
+		try {
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(output);
+			assertEquals(200, http.getStatusLine().getStatusCode());
+		} finally {
+			IOUtils.closeQuietly(http);;
+		}
+	}
+
 
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/SystemProviderDstu3Test.java

@@ -129,7 +129,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		
 		CloseableHttpResponse resp = ourHttpClient.execute(req);
 		try {
-			String encoded = IOUtils.toString(resp.getEntity().getContent());
+			String encoded = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(encoded);
 
 			assertThat(encoded, containsString("transaction-response")); 
@@ -162,7 +162,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		
 		CloseableHttpResponse resp = ourHttpClient.execute(req);
 		try {
-			String encoded = IOUtils.toString(resp.getEntity().getContent());
+			String encoded = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(encoded);
 
 			assertThat(encoded, containsString("transaction-response"));
@@ -271,14 +271,14 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		try {
 	
 			InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream("/questionnaire-sdc-profile-example-ussg-fht.xml");
-			String bundleStr = IOUtils.toString(bundleRes);
+			String bundleStr = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 	
 			HttpPost req = new HttpPost(ourServerBase);
 			req.setEntity(new StringEntity(bundleStr, ContentType.parse(Constants.CT_FHIR_XML + "; charset=utf-8")));
 			
 			CloseableHttpResponse resp = ourHttpClient.execute(req);
 			try {
-				String encoded = IOUtils.toString(resp.getEntity().getContent());
+				String encoded = IOUtils.toString(resp.getEntity().getContent(), StandardCharsets.UTF_8);
 				ourLog.info(encoded);
 	
 				//@formatter:off
@@ -314,7 +314,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 
 		try {
-			String response = IOUtils.toString(http.getEntity().getContent());
+			String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(response);
 			assertThat(response, containsString("_format=json"));
 			assertEquals(200, http.getStatusLine().getStatusCode());
@@ -342,7 +342,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		get.addHeader("Accept", "application/xml+fhir");
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
-			String response = IOUtils.toString(http.getEntity().getContent());
+			String response = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(response);
 			assertThat(response, not(containsString("_format")));
 			assertEquals(200, http.getStatusLine().getStatusCode());
@@ -373,7 +373,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		HttpGet get = new HttpGet(ourServerBase + "/$mark-all-resources-for-reindexing");
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertEquals(200, http.getStatusLine().getStatusCode());
 		} finally {
@@ -404,7 +404,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
 			assertEquals(200, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 
 			Parameters parameters = ourCtx.newXmlParser().parseResource(Parameters.class, output);
@@ -434,7 +434,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		CloseableHttpResponse http = ourHttpClient.execute(get);
 		try {
 			assertEquals(400, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertThat(output, containsString("Parameter 'context' must be provided"));
 		} finally {
@@ -445,7 +445,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		http = ourHttpClient.execute(get);
 		try {
 			assertEquals(400, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertThat(output, containsString("Parameter 'searchParam' must be provided"));
 		} finally {
@@ -456,7 +456,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		http = ourHttpClient.execute(get);
 		try {
 			assertEquals(400, http.getStatusLine().getStatusCode());
-			String output = IOUtils.toString(http.getEntity().getContent());
+			String output = IOUtils.toString(http.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(output);
 			assertThat(output, containsString("Parameter 'text' must be provided"));
 		} finally {
@@ -500,7 +500,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 	public void testTransactionFromBundle2() throws Exception {
 
 		InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 
@@ -516,7 +516,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 		 */
 
 		bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/transaction_link_patient_eve_temp.xml");
-		bundle = IOUtils.toString(bundleRes);
+		bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 
@@ -540,7 +540,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 	public void testTransactionFromBundle3() throws Exception {
 
 		InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/grahame-transaction.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 	}
@@ -548,7 +548,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 	@Test
 	public void testTransactionFromBundle4() throws Exception {
 		InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/simone_bundle.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		String response = ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		ourLog.info(response);
 		Bundle bundleResp = ourCtx.newXmlParser().parseResource(Bundle.class, response);
@@ -563,7 +563,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 	@Test
 	public void testTransactionFromBundle5() throws Exception {
 		InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/simone_bundle2.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		try {
 			ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 			fail();
@@ -577,7 +577,7 @@ public class SystemProviderDstu3Test extends BaseJpaDstu3Test {
 	@Test
 	public void testTransactionFromBundle6() throws Exception {
 		InputStream bundleRes = SystemProviderDstu3Test.class.getResourceAsStream("/simone_bundle3.xml");
-		String bundle = IOUtils.toString(bundleRes);
+		String bundle = IOUtils.toString(bundleRes, StandardCharsets.UTF_8);
 		ourClient.transaction().withBundle(bundle).prettyPrint().execute();
 		// try {
 		// fail();