Always apply new search params to any resource reindexing

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -174,20 +174,14 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
 	@Autowired
 	private ISearchDao mySearchDao;
 	@Autowired
-	private ISearchParamExtractor mySearchParamExtractor;
-	@Autowired
 	private ISearchParamPresenceSvc mySearchParamPresenceSvc;
 	//@Autowired
 	//private ISearchResultDao mySearchResultDao;
 	@Autowired
-	private IResourceIndexedCompositeStringUniqueDao myResourceIndexedCompositeStringUniqueDao;
-	@Autowired
 	private BeanFactory beanFactory;
 	@Autowired
 	private DaoRegistry myDaoRegistry;
 	@Autowired
-	private SearchParamExtractorService mySearchParamExtractorService;
-	@Autowired
 	private SearchParamWithInlineReferencesExtractor mySearchParamWithInlineReferencesExtractor;
 	@Autowired
 	private DatabaseSearchParamSynchronizer myDatabaseSearchParamSynchronizer;

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/reindex/IResourceReindexingSvc.java

@@ -58,4 +58,6 @@ public interface IResourceReindexingSvc {
 	 * to be used by unit tests.
 	 */
 	void cancelAndPurgeAllJobs();
+
+	int countReindexJobs();
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/reindex/ResourceReindexingSvcImpl.java

@@ -30,11 +30,11 @@ import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceHistoryTableDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceReindexJobDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
-import ca.uhn.fhir.jpa.model.entity.ForcedId;
 import ca.uhn.fhir.jpa.entity.ResourceReindexJobEntity;
+import ca.uhn.fhir.jpa.model.entity.ForcedId;
 import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
-import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
 import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
 import ca.uhn.fhir.util.StopWatch;
 import com.google.common.annotations.VisibleForTesting;
@@ -203,7 +203,7 @@ public class ResourceReindexingSvcImpl implements IResourceReindexingSvc {
 		return null;
 	}
 
-	private Integer doReindexingPassInsideLock() {
+	private int doReindexingPassInsideLock() {
 		expungeJobsMarkedAsDeleted();
 		return runReindexJobs();
 	}
@@ -232,14 +232,17 @@ public class ResourceReindexingSvcImpl implements IResourceReindexingSvc {
 		expungeJobsMarkedAsDeleted();
 	}
 
+	@Autowired
+	private ISearchParamRegistry mySearchParamRegistry;
+
 	private int runReindexJobs() {
-		Collection<ResourceReindexJobEntity> jobs = myTxTemplate.execute(t -> myReindexJobDao.findAll(PageRequest.of(0, 10), false));
+		Collection<ResourceReindexJobEntity> jobs = getResourceReindexJobEntities();
-		assert jobs != null;
 
 		if (jobs.size() > 0) {
 			ourLog.info("Running {} reindex jobs: {}", jobs.size(), jobs);
 		} else {
 			ourLog.debug("Running {} reindex jobs: {}", jobs.size(), jobs);
+			return 0;
 		}
 
 		int count = 0;
@@ -255,6 +258,17 @@ public class ResourceReindexingSvcImpl implements IResourceReindexingSvc {
 		return count;
 	}
 
+	@Override
+	public int countReindexJobs() {
+		return getResourceReindexJobEntities().size();
+	}
+
+	private Collection<ResourceReindexJobEntity> getResourceReindexJobEntities() {
+		Collection<ResourceReindexJobEntity> jobs = myTxTemplate.execute(t -> myReindexJobDao.findAll(PageRequest.of(0, 10), false));
+		assert jobs != null;
+		return jobs;
+	}
+
 	private void markJobAsDeleted(ResourceReindexJobEntity theJob) {
 		ourLog.info("Marking reindexing job ID[{}] as deleted", theJob.getId());
 		myTxTemplate.execute(t -> {
@@ -274,6 +288,10 @@ public class ResourceReindexingSvcImpl implements IResourceReindexingSvc {
 		StopWatch sw = new StopWatch();
 		AtomicInteger counter = new AtomicInteger();
 
+		if (theJob.getThresholdLow() == null) {
+			mySearchParamRegistry.forceRefresh();
+		}
+
 		// Calculate range
 		Date low = theJob.getThresholdLow() != null ? theJob.getThresholdLow() : BEGINNING_OF_TIME;
 		Date high = theJob.getThresholdHigh();
@@ -461,7 +479,7 @@ public class ResourceReindexingSvcImpl implements IResourceReindexingSvc {
 
 						IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceTable.getResourceType());
 						long expectedVersion = resourceTable.getVersion();
-						IBaseResource resource = dao.read(resourceTable.getIdDt().toVersionless(), null,true);
+						IBaseResource resource = dao.read(resourceTable.getIdDt().toVersionless(), null, true);
 						if (resource == null) {
 							throw new InternalErrorException("Could not find resource version " + resourceTable.getIdDt().toUnqualified().getValue() + " in database");
 						}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java

@@ -10,6 +10,7 @@ import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
 import ca.uhn.fhir.util.TestUtil;
+import com.google.common.collect.Lists;
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.dstu3.model.CodeSystem;
 import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode;
@@ -23,6 +24,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.*;
@@ -583,6 +585,50 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
 		}
 	}
 
+	/**
+	 * Check that a custom ValueSet against a custom CodeSystem expands correctly
+	 */
+	@Test
+	public void testCustomValueSetExpansion() {
+
+		CodeSystem cs=  new CodeSystem();
+		cs.setUrl("http://codesystems-r-us");
+		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
+		IIdType csId = myCodeSystemDao.create(cs).getId().toUnqualifiedVersionless();
+
+		TermCodeSystemVersion version = new TermCodeSystemVersion();
+		version.getConcepts().add(new TermConcept(version, "A"));
+		version.getConcepts().add(new TermConcept(version, "B"));
+		version.getConcepts().add(new TermConcept(version, "C"));
+		version.getConcepts().add(new TermConcept(version, "D"));
+		runInTransaction(()->{
+			ResourceTable resTable = myEntityManager.find(ResourceTable.class, csId.getIdPartAsLong());
+			version.setResource(resTable);
+			myTermSvc.storeNewCodeSystemVersion(csId.getIdPartAsLong(), cs.getUrl(), "My System", version);
+		});
+
+		org.hl7.fhir.dstu3.model.ValueSet vs = new org.hl7.fhir.dstu3.model.ValueSet();
+		vs.setUrl("http://valuesets-r-us");
+		vs.getCompose()
+			.addInclude()
+			.setSystem(cs.getUrl())
+			.addConcept(new org.hl7.fhir.dstu3.model.ValueSet.ConceptReferenceComponent().setCode("A"))
+			.addConcept(new org.hl7.fhir.dstu3.model.ValueSet.ConceptReferenceComponent().setCode("C"));
+		myValueSetDao.create(vs);
+
+		org.hl7.fhir.dstu3.model.ValueSet expansion = myValueSetDao.expandByIdentifier(vs.getUrl(), null);
+		List<String> expansionCodes = expansion
+			.getExpansion()
+			.getContains()
+			.stream()
+			.map(t -> t.getCode())
+			.sorted()
+			.collect(Collectors.toList());
+		assertEquals(Lists.newArrayList("A","C"), expansionCodes);
+
+	}
+
+
 	public static List<String> toCodesContains(List<ValueSet.ValueSetExpansionContainsComponent> theContains) {
 		List<String> retVal = new ArrayList<>();
 

hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/BaseSearchParamRegistry.java

@@ -57,10 +57,9 @@ public abstract class BaseSearchParamRegistry<SP extends IBaseResource> implemen
 	@Autowired
 	private ModelConfig myModelConfig;
 	private volatile long myLastRefresh;
-	private ApplicationContext myApplicationContext;
 	private ISearchParamProvider mySearchParamProvider;
 
-	public BaseSearchParamRegistry(ISearchParamProvider theSearchParamProvider) {
+	BaseSearchParamRegistry(ISearchParamProvider theSearchParamProvider) {
 		super();
 		mySearchParamProvider = theSearchParamProvider;
 	}
@@ -128,7 +127,7 @@ public abstract class BaseSearchParamRegistry<SP extends IBaseResource> implemen
 		return Collections.unmodifiableList(retVal);
 	}
 
-	public Map<String, Map<String, RuntimeSearchParam>> getBuiltInSearchParams() {
+	private Map<String, Map<String, RuntimeSearchParam>> getBuiltInSearchParams() {
 		return myBuiltInSearchParams;
 	}
 

src/changes/changes.xml

@@ -151,6 +151,11 @@
 				encounters new extensions. Thanks to Heinz-Dieter Conradi for the
 				pull request!
 			</action>
+			<action type="fix">
+				Under some circumstances, when a custom search parameter was added to the JPA server 
+				resources could start reindexing before the new search parameter had been saved, meaning that
+				it was not applied to all resources. This has been corrected.
+			</action>
 		</release>
 		<release version="3.6.0" date="2018-11-12" description="Food">
 			<action type="add">