diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java index bf46ed1f171..f566a3a7aff 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java @@ -25,7 +25,7 @@ public final class Msg { /** * IMPORTANT: Please update the following comment after you add a new code - * Last code value: 2129 + * Last code value: 2130 */ private Msg() {} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjection.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjection.java index dd7dcde57d2..73f2519b358 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjection.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjection.java @@ -20,8 +20,10 @@ package ca.uhn.fhir.jpa.dao.search; * #L% */ +import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.parser.IParser; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -29,12 +31,16 @@ import org.hl7.fhir.instance.model.api.IBaseResource; * Query result when fetching full resources from HSearch. */ public class ExtendedHSearchResourceProjection { + public static final String RESOURCE_NOT_STORED_ERROR = "Resource not stored in search index: "; + final long myPid; final String myForcedId; final String myResourceString; public ExtendedHSearchResourceProjection(long thePid, String theForcedId, String theResourceString) { - Validate.notEmpty(theResourceString, "Resource not stored in search index: " + thePid); + if (StringUtils.isEmpty(theResourceString)) { + throw new ResourceNotFoundInIndexException(Msg.code(2130) + RESOURCE_NOT_STORED_ERROR + thePid); + } myPid = thePid; myForcedId = theForcedId; myResourceString = theResourceString; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ResourceNotFoundInIndexException.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ResourceNotFoundInIndexException.java new file mode 100644 index 00000000000..a72bce7f654 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ResourceNotFoundInIndexException.java @@ -0,0 +1,18 @@ +package ca.uhn.fhir.jpa.dao.search; + +public class ResourceNotFoundInIndexException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + public ResourceNotFoundInIndexException(Throwable theCause) { + super(theCause.getMessage(), theCause); + } + + public ResourceNotFoundInIndexException(String theMessage) { + super(theMessage); + } + + public ResourceNotFoundInIndexException(String theString, Throwable theCause) { + super(theString, theCause); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index d6c7b78299b..a30e0b591be 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -35,6 +35,7 @@ import ca.uhn.fhir.jpa.dao.BaseStorageDao; import ca.uhn.fhir.jpa.dao.IResultIterator; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; +import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchInclude; import ca.uhn.fhir.jpa.entity.SearchTypeEnum; @@ -42,7 +43,6 @@ import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; -import ca.uhn.fhir.jpa.search.builder.predicate.ResourceLinkPredicateBuilder; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum; @@ -365,7 +365,14 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { if (mySearchStrategyFactory.isSupportsHSearchDirect(theResourceType, theParams, theRequestDetails)) { ourLog.info("Search {} is using direct load strategy", searchUuid); SearchStrategyFactory.ISearchStrategy direct = mySearchStrategyFactory.makeDirectStrategy(searchUuid, theResourceType, theParams, theRequestDetails); - return direct.get(); + + try { + return direct.get(); + + } catch (ResourceNotFoundInIndexException theE) { + // some resources were not found in index, so we will inform this and resort to JPA search + ourLog.warn("Some resources were not found in index. Make sure all resources were indexed. Resorting to database search."); + } } ourLog.debug("Search {} is loading in synchronous mode", searchUuid); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 0ce619cdf37..0f2b46ea73c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -43,6 +43,7 @@ import ca.uhn.fhir.jpa.dao.IResultIterator; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao; import ca.uhn.fhir.jpa.dao.data.IResourceTagDao; +import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException; import ca.uhn.fhir.jpa.entity.ResourceSearchView; import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; import ca.uhn.fhir.jpa.model.config.PartitionSettings; @@ -118,7 +119,6 @@ import javax.persistence.criteria.From; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -953,11 +953,18 @@ public class SearchBuilder implements ISearchBuilder { // Can we fast track this loading by checking elastic search? if (isLoadingFromElasticSearchSupported(thePids)) { - theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids)); - } else { - // We only chunk because some jdbc drivers can't handle long param lists. - new QueryChunker().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position)); + try { + theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids)); + return; + + } catch (ResourceNotFoundInIndexException theE) { + // some resources were not found in index, so we will inform this and resort to JPA search + ourLog.warn("Some resources were not found in index. Make sure all resources were indexed. Resorting to database search."); + } } + + // We only chunk because some jdbc drivers can't handle long param lists. + new QueryChunker().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position)); } /** diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java index ab7789f7d5c..4470141583c 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java +++ b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java @@ -731,6 +731,34 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl } } + /** + * When configuring direct resource load for populated index a full reindex is required + * Code should detect the case and instead of throwing "Resource not stored in search index" exception, should log + * a warning and route the strategy to be rerun in JPA. + * This test validates that behaviour. + */ + @Test + public void testDirectPathWholeResourceNotIndexedWorks() { + IIdType id1 = myTestDataBuilder.createObservation(List.of(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"))); + + // set it after creating resource, so search doesn't find it in the index + myDaoConfig.setStoreResourceInHSearchIndex(true); + + myCaptureQueriesListener.clear(); + + List result = searchForFastResources("Observation?code=theCode"); + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + + assertThat(result, hasSize(1)); + assertEquals( ((Observation) result.get(0)).getIdElement().getIdPart(), id1.getIdPart()); + assertEquals(2, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "JPA search for IDs and for resources"); + + // restore changed property + DaoConfig defaultConfig = new DaoConfig(); + myDaoConfig.setStoreResourceInHSearchIndex(defaultConfig.isStoreResourceInHSearchIndex()); + } + + @Test public void testExpandWithIsAInExternalValueSet() { createExternalCsAndLocalVs(); @@ -2056,10 +2084,10 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl @Test public void byDate() { // check milli level precision - String id1 = createObservation(withId("20-000"), withEffectiveDate("2017-01-20T03:21:47.000")).getIdPart(); - String id2 = createObservation(withId("24-002"), withEffectiveDate("2017-01-24T03:21:47.002")).getIdPart(); - String id3 = createObservation(withId("24-001"), withEffectiveDate("2017-01-24T03:21:47.001")).getIdPart(); - String id4 = createObservation(withId("20-002"), withEffectiveDate("2017-01-20T03:21:47.002")).getIdPart(); + String id1 = createObservation(List.of(withId("20-000"), withEffectiveDate("2017-01-20T03:21:47.000"))).getIdPart(); + String id2 = createObservation(List.of(withId("24-002"), withEffectiveDate("2017-01-24T03:21:47.002"))).getIdPart(); + String id3 = createObservation(List.of(withId("24-001"), withEffectiveDate("2017-01-24T03:21:47.001"))).getIdPart(); + String id4 = createObservation(List.of(withId("20-002"), withEffectiveDate("2017-01-20T03:21:47.002"))).getIdPart(); myCaptureQueriesListener.clear(); List result = myTestDaoSearch.searchForIds("/Observation?_sort=-date"); diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjectionTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjectionTest.java index cb2a502c9a1..164a940d124 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjectionTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchResourceProjectionTest.java @@ -1,14 +1,17 @@ package ca.uhn.fhir.jpa.dao.search; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.parser.IParser; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Observation; import org.junit.jupiter.api.Test; +import static ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection.RESOURCE_NOT_STORED_ERROR; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; class ExtendedHSearchResourceProjectionTest { final FhirContext myFhirContext = FhirContext.forR4(); @@ -37,4 +40,24 @@ class ExtendedHSearchResourceProjectionTest { } + @Test + public void nullResourceStringThrows() { + ResourceNotFoundInIndexException ex = assertThrows( + ResourceNotFoundInIndexException.class, + () -> new ExtendedHSearchResourceProjection(22, null, null)); + assertThat(ex.getMessage(), equalTo(Msg.code(2130) + RESOURCE_NOT_STORED_ERROR + "22")); + } + + + @Test + public void emptyResourceStringThrows() { + ResourceNotFoundInIndexException ex = assertThrows( + ResourceNotFoundInIndexException.class, + () -> new ExtendedHSearchResourceProjection(22, null, "")); + assertThat(ex.getMessage(), equalTo(Msg.code(2130) + RESOURCE_NOT_STORED_ERROR + "22")); + } + + + + }