Add top level support for ElasticSearch (#1514)

* ElasticSearch work

* Add ElasticSearch properties builder

* Clean up POM

* Remove redundant project

* Try to troubleshoot embedded elasticsearch

* Another test attempt

* Add credentials to elasticsearch config

* Work on lastn

* Address review comments

* A couple of test fixes

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java

@@ -237,6 +237,10 @@ public class Constants {
 	public static final int STATUS_HTTP_202_ACCEPTED = 202;
 	public static final String HEADER_X_PROGRESS = "X-Progress";
 	public static final String HEADER_RETRY_AFTER = "Retry-After";
+	/**
+	 * Operation name for the $lastn operation
+	 */
+	public static final String OPERATION_LASTN = "$lastn";
 
 	static {
 		CHARSET_UTF8 = StandardCharsets.UTF_8;

hapi-fhir-jpaserver-base/pom.xml

@@ -483,6 +483,8 @@
 			<groupId>org.glassfish</groupId>
 			<artifactId>javax.el</artifactId>
 		</dependency>
+
+		<!-- Hibernate Search -->
 		<dependency>
 			<groupId>org.hibernate</groupId>
 			<artifactId>hibernate-search-orm</artifactId>
@@ -495,6 +497,10 @@
 			<groupId>org.apache.lucene</groupId>
 			<artifactId>lucene-analyzers-phonetic</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.hibernate</groupId>
+			<artifactId>hibernate-search-elasticsearch</artifactId>
+		</dependency>
 
 		<!-- Misc -->
 		<dependency>
@@ -568,6 +574,11 @@
 			<artifactId>greenmail-spring</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>pl.allegro.tech</groupId>
+			<artifactId>embedded-elasticsearch</artifactId>
+			<version>2.10.0</version>
+		</dependency>
 
 		<dependency>
 			<groupId>com.github.ben-manes.caffeine</groupId>

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceSearchView.java

@@ -62,17 +62,13 @@ public class ResourceSearchView implements IBaseResourceEntity, Serializable {
 	@Id
 	@Column(name = "PID")
 	private Long myId;
-
 	@Column(name = "RES_ID")
 	private Long myResourceId;
-
 	@Column(name = "RES_TYPE", length = Constants.MAX_RESOURCE_NAME_LENGTH)
 	private String myResourceType;
-
 	@Column(name = "RES_VERSION")
 	@Enumerated(EnumType.STRING)
 	private FhirVersionEnum myFhirVersion;
-
 	@Column(name = "RES_VER")
 	private Long myResourceVersion;
 	@Column(name = "PROV_REQUEST_ID", length = Constants.REQUEST_ID_LENGTH)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchHibernatePropertiesBuilder.java

@@ -0,0 +1,90 @@
+package ca.uhn.fhir.jpa.search.elastic;
+
+import org.hibernate.search.cfg.Environment;
+import org.hibernate.search.elasticsearch.cfg.ElasticsearchEnvironment;
+import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus;
+import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy;
+
+import java.util.Properties;
+
+/**
+ * This class is used to inject appropriate properties into a hibernate
+ * Properties object being used to create an entitymanager for a HAPI
+ * FHIR JPA server.
+ */
+public class ElasticsearchHibernatePropertiesBuilder {
+
+	private ElasticsearchIndexStatus myRequiredIndexStatus = ElasticsearchIndexStatus.YELLOW;
+	private String myRestUrl;
+	private String myUsername;
+	private String myPassword;
+	private IndexSchemaManagementStrategy myIndexSchemaManagementStrategy = IndexSchemaManagementStrategy.CREATE;
+	private long myIndexManagementWaitTimeoutMillis = 10000L;
+	private boolean myDebugRefreshAfterWrite = false;
+	private boolean myDebugPrettyPrintJsonLog = false;
+
+	public ElasticsearchHibernatePropertiesBuilder setUsername(String theUsername) {
+		myUsername = theUsername;
+		return this;
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setPassword(String thePassword) {
+		myPassword = thePassword;
+		return this;
+	}
+
+	public void apply(Properties theProperties) {
+
+		// Don't use the Lucene properties as they conflict
+		theProperties.remove("hibernate.search.model_mapping");
+
+		// the below properties are used for ElasticSearch integration
+		theProperties.put("hibernate.search.default." + Environment.INDEX_MANAGER_IMPL_NAME, "elasticsearch");
+		theProperties.put("hibernate.search." + ElasticsearchEnvironment.ANALYSIS_DEFINITION_PROVIDER, ElasticsearchMappingProvider.class.getName());
+
+		theProperties.put("hibernate.search.default.elasticsearch.host", myRestUrl);
+		theProperties.put("hibernate.search.default.elasticsearch.username", myUsername);
+		theProperties.put("hibernate.search.default.elasticsearch.password", myPassword);
+
+		theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.INDEX_SCHEMA_MANAGEMENT_STRATEGY, myIndexSchemaManagementStrategy.getExternalName());
+		theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.INDEX_MANAGEMENT_WAIT_TIMEOUT, Long.toString(myIndexManagementWaitTimeoutMillis));
+		theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.REQUIRED_INDEX_STATUS, myRequiredIndexStatus.getElasticsearchString());
+
+		// Only for unit tests
+		theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.REFRESH_AFTER_WRITE, Boolean.toString(myDebugRefreshAfterWrite));
+		theProperties.put("hibernate.search." + ElasticsearchEnvironment.LOG_JSON_PRETTY_PRINTING, Boolean.toString(myDebugPrettyPrintJsonLog));
+
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setRequiredIndexStatus(ElasticsearchIndexStatus theRequiredIndexStatus) {
+		myRequiredIndexStatus = theRequiredIndexStatus;
+		return this;
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setRestUrl(String theRestUrl) {
+		myRestUrl = theRestUrl;
+		return this;
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy theIndexSchemaManagementStrategy) {
+		myIndexSchemaManagementStrategy = theIndexSchemaManagementStrategy;
+		return this;
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setIndexManagementWaitTimeoutMillis(long theIndexManagementWaitTimeoutMillis) {
+		myIndexManagementWaitTimeoutMillis = theIndexManagementWaitTimeoutMillis;
+		return this;
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setDebugRefreshAfterWrite(boolean theDebugRefreshAfterWrite) {
+		myDebugRefreshAfterWrite = theDebugRefreshAfterWrite;
+		return this;
+	}
+
+	public ElasticsearchHibernatePropertiesBuilder setDebugPrettyPrintJsonLog(boolean theDebugPrettyPrintJsonLog) {
+		myDebugPrettyPrintJsonLog = theDebugPrettyPrintJsonLog;
+		return this;
+	}
+
+
+}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchMappingProvider.java

@@ -1,4 +1,4 @@
-package ca.uhn.fhir.jpa.search;
+package ca.uhn.fhir.jpa.search.elastic;
 
 /*-
  * #%L
@@ -20,9 +20,8 @@ package ca.uhn.fhir.jpa.search;
  * #L%
  */
 
-import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
-import org.hibernate.search.elasticsearch.analyzer.definition.ElasticsearchAnalysisDefinitionRegistryBuilder;
 import org.hibernate.search.elasticsearch.analyzer.definition.ElasticsearchAnalysisDefinitionProvider;
+import org.hibernate.search.elasticsearch.analyzer.definition.ElasticsearchAnalysisDefinitionRegistryBuilder;
 
 public class ElasticsearchMappingProvider implements ElasticsearchAnalysisDefinitionProvider {
 
@@ -39,10 +38,7 @@ public class ElasticsearchMappingProvider implements ElasticsearchAnalysisDefini
 
 		builder.analyzer("autocompletePhoneticAnalyzer")
 			.withTokenizer("standard")
-			.withTokenFilters("standard", "stop", "snowball_english", "phonetic_doublemetaphone");
+			.withTokenFilters("standard", "stop", "snowball_english");
-		builder.tokenFilter("phonetic_doublemetaphone")
-			.type("phonetic")
-			.param("encoder", "double_metaphone");
 		builder.tokenFilter("snowball_english").type("snowball").param("language", "English");
 
 		builder.analyzer("autocompleteNGramAnalyzer")

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3WithoutLuceneConfig.java

@@ -1,46 +0,0 @@
-package ca.uhn.fhir.jpa.config;
-
-import java.util.Properties;
-
-import org.hibernate.dialect.H2Dialect;
-import org.hibernate.jpa.HibernatePersistenceProvider;
-import org.springframework.beans.factory.annotation.Autowire;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
-import org.springframework.transaction.annotation.EnableTransactionManagement;
-
-import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl;
-import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
-
-@Configuration
-@EnableTransactionManagement()
-public class TestDstu3WithoutLuceneConfig extends TestDstu3Config {
-
-	/**
-	 * Disable fulltext searching
-	 */
-	@Override
-	public IFulltextSearchSvc searchDaoDstu3() {
-		return null;
-	}
-
-	@Override
-	@Bean
-	public LocalContainerEntityManagerFactoryBean entityManagerFactory() {
-		LocalContainerEntityManagerFactoryBean retVal = super.entityManagerFactory();
-		retVal.setJpaProperties(jpaProperties());
-		return retVal;
-	}
-
-	private Properties jpaProperties() {
-		Properties extraProperties = new Properties();
-		extraProperties.put("hibernate.format_sql", "false");
-		extraProperties.put("hibernate.show_sql", "false");
-		extraProperties.put("hibernate.hbm2ddl.auto", "update");
-		extraProperties.put("hibernate.dialect", H2Dialect.class.getName());
-		extraProperties.put("hibernate.search.autoregister_listeners", "false");
-		return extraProperties;
-	}
-
-}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java

@@ -138,7 +138,8 @@ public class TestR4Config extends BaseJavaConfigR4 {
 		return retVal;
 	}
 
-	private Properties jpaProperties() {
+	@Bean
+	public Properties jpaProperties() {
 		Properties extraProperties = new Properties();
 		extraProperties.put("hibernate.format_sql", "false");
 		extraProperties.put("hibernate.show_sql", "false");

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4ConfigWithElasticSearch.java

@@ -0,0 +1,74 @@
+package ca.uhn.fhir.jpa.config;
+
+import ca.uhn.fhir.context.ConfigurationException;
+import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder;
+import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus;
+import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic;
+import pl.allegro.tech.embeddedelasticsearch.PopularProperties;
+
+import javax.annotation.PreDestroy;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+@Configuration
+public class TestR4ConfigWithElasticSearch extends TestR4Config {
+
+	private static final Logger ourLog = LoggerFactory.getLogger(TestR4ConfigWithElasticSearch.class);
+	private static final String ELASTIC_VERSION = "6.5.4";
+
+	@Override
+	@Bean
+	public Properties jpaProperties() {
+		Properties retVal = super.jpaProperties();
+
+		// Force elasticsearch to start first
+		int httpPort = embeddedElasticSearch().getHttpPort();
+		ourLog.info("ElasticSearch started on port: {}", httpPort);
+
+		new ElasticsearchHibernatePropertiesBuilder()
+			.setDebugRefreshAfterWrite(true)
+			.setDebugPrettyPrintJsonLog(true)
+			.setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE)
+			.setIndexManagementWaitTimeoutMillis(10000)
+			.setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW)
+			.setRestUrl("http://localhost:" + httpPort)
+			.setUsername("")
+			.setPassword("")
+			.apply(retVal);
+
+		return retVal;
+	}
+
+	@Bean
+	public EmbeddedElastic embeddedElasticSearch() {
+		EmbeddedElastic embeddedElastic = null;
+		try {
+			embeddedElastic = EmbeddedElastic.builder()
+				.withElasticVersion(ELASTIC_VERSION)
+				.withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0)
+				.withSetting(PopularProperties.HTTP_PORT, 0)
+				.withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID())
+				.withStartTimeout(60, TimeUnit.SECONDS)
+				.build()
+				.start();
+		} catch (IOException | InterruptedException e) {
+			throw new ConfigurationException(e);
+		}
+
+		return embeddedElastic;
+	}
+
+
+	@PreDestroy
+	public void stop() {
+		embeddedElasticSearch().stop();
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4WithLuceneDisabledConfig.java

@@ -15,7 +15,7 @@ import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
 
 @Configuration
 @EnableTransactionManagement()
-public class TestR4WithoutLuceneConfig extends TestR4Config {
+public class TestR4WithLuceneDisabledConfig extends TestR4Config {
 
 	/**
 	 * Disable fulltext searching
@@ -34,7 +34,8 @@ public class TestR4WithoutLuceneConfig extends TestR4Config {
 		return retVal;
 	}
 
-	private Properties jpaProperties() {
+	@Override
+	public Properties jpaProperties() {
 		Properties extraProperties = new Properties();
 		extraProperties.put("hibernate.format_sql", "false");
 		extraProperties.put("hibernate.show_sql", "false");

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchWithLuceneDisabledTest.java

@@ -1,234 +0,0 @@
-package ca.uhn.fhir.jpa.dao.dstu3;
-
-import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc;
-import ca.uhn.fhir.jpa.config.TestDstu3WithoutLuceneConfig;
-import ca.uhn.fhir.jpa.dao.*;
-import ca.uhn.fhir.jpa.provider.dstu3.JpaSystemProviderDstu3;
-import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc;
-import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
-import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
-import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
-import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
-import ca.uhn.fhir.rest.param.StringParam;
-import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
-import ca.uhn.fhir.util.TestUtil;
-import org.hl7.fhir.dstu3.hapi.ctx.IValidationSupport;
-import org.hl7.fhir.dstu3.model.*;
-import org.hl7.fhir.instance.model.api.IIdType;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-import org.springframework.transaction.PlatformTransactionManager;
-
-import javax.persistence.EntityManager;
-
-import static org.junit.Assert.*;
-
-// @RunWith(SpringJUnit4ClassRunner.class)
-// @ContextConfiguration(classes= {TestDstu3WithoutLuceneConfig.class})
-// @SuppressWarnings("unchecked")
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes = {TestDstu3WithoutLuceneConfig.class})
-public class FhirResourceDaoDstu3SearchWithLuceneDisabledTest extends BaseJpaTest {
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoDstu3SearchWithLuceneDisabledTest.class);
-	@Autowired
-	protected DaoConfig myDaoConfig;
-	@Autowired
-	protected PlatformTransactionManager myTxManager;
-	@Autowired
-	protected ISearchParamPresenceSvc mySearchParamPresenceSvc;
-	@Autowired
-	protected ISearchCoordinatorSvc mySearchCoordinatorSvc;
-	@Autowired
-	protected ISearchParamRegistry mySearchParamRegistry;
-	@Autowired
-	@Qualifier("myAllergyIntoleranceDaoDstu3")
-	private IFhirResourceDao<AllergyIntolerance> myAllergyIntoleranceDao;
-	@Autowired
-	@Qualifier("myAppointmentDaoDstu3")
-	private IFhirResourceDao<Appointment> myAppointmentDao;
-	@Autowired
-	@Qualifier("myAuditEventDaoDstu3")
-	private IFhirResourceDao<AuditEvent> myAuditEventDao;
-	@Autowired
-	@Qualifier("myBundleDaoDstu3")
-	private IFhirResourceDao<Bundle> myBundleDao;
-	@Autowired
-	@Qualifier("myCarePlanDaoDstu3")
-	private IFhirResourceDao<CarePlan> myCarePlanDao;
-	@Autowired
-	@Qualifier("myCodeSystemDaoDstu3")
-	private IFhirResourceDao<CodeSystem> myCodeSystemDao;
-	@Autowired
-	@Qualifier("myCompartmentDefinitionDaoDstu3")
-	private IFhirResourceDao<CompartmentDefinition> myCompartmentDefinitionDao;
-	@Autowired
-	@Qualifier("myConceptMapDaoDstu3")
-	private IFhirResourceDao<ConceptMap> myConceptMapDao;
-	@Autowired
-	@Qualifier("myConditionDaoDstu3")
-	private IFhirResourceDao<Condition> myConditionDao;
-	@Autowired
-	@Qualifier("myDeviceDaoDstu3")
-	private IFhirResourceDao<Device> myDeviceDao;
-	@Autowired
-	@Qualifier("myDiagnosticReportDaoDstu3")
-	private IFhirResourceDao<DiagnosticReport> myDiagnosticReportDao;
-	@Autowired
-	@Qualifier("myEncounterDaoDstu3")
-	private IFhirResourceDao<Encounter> myEncounterDao;
-	// @PersistenceContext()
-	@Autowired
-	private EntityManager myEntityManager;
-	@Autowired
-	private FhirContext myFhirCtx;
-	@Autowired
-	@Qualifier("myImmunizationDaoDstu3")
-	private IFhirResourceDao<Immunization> myImmunizationDao;
-	@Autowired
-	@Qualifier("myLocationDaoDstu3")
-	private IFhirResourceDao<Location> myLocationDao;
-	@Autowired
-	@Qualifier("myMediaDaoDstu3")
-	private IFhirResourceDao<Media> myMediaDao;
-	@Autowired
-	@Qualifier("myMedicationDaoDstu3")
-	private IFhirResourceDao<Medication> myMedicationDao;
-	@Autowired
-	@Qualifier("myMedicationRequestDaoDstu3")
-	private IFhirResourceDao<MedicationRequest> myMedicationRequestDao;
-	@Autowired
-	@Qualifier("myNamingSystemDaoDstu3")
-	private IFhirResourceDao<NamingSystem> myNamingSystemDao;
-	@Autowired
-	@Qualifier("myObservationDaoDstu3")
-	private IFhirResourceDao<Observation> myObservationDao;
-	@Autowired
-	@Qualifier("myOperationDefinitionDaoDstu3")
-	private IFhirResourceDao<OperationDefinition> myOperationDefinitionDao;
-	@Autowired
-	@Qualifier("myOrganizationDaoDstu3")
-	private IFhirResourceDao<Organization> myOrganizationDao;
-	@Autowired
-	@Qualifier("myPatientDaoDstu3")
-	private IFhirResourceDaoPatient<Patient> myPatientDao;
-	@Autowired
-	@Qualifier("myPractitionerDaoDstu3")
-	private IFhirResourceDao<Practitioner> myPractitionerDao;
-	@Autowired
-	@Qualifier("myQuestionnaireDaoDstu3")
-	private IFhirResourceDao<Questionnaire> myQuestionnaireDao;
-	@Autowired
-	@Qualifier("myQuestionnaireResponseDaoDstu3")
-	private IFhirResourceDao<QuestionnaireResponse> myQuestionnaireResponseDao;
-	@Autowired
-	@Qualifier("myResourceProvidersDstu3")
-	private Object myResourceProviders;
-	@Autowired
-	@Qualifier("myStructureDefinitionDaoDstu3")
-	private IFhirResourceDao<StructureDefinition> myStructureDefinitionDao;
-	@Autowired
-	@Qualifier("mySubscriptionDaoDstu3")
-	private IFhirResourceDaoSubscription<Subscription> mySubscriptionDao;
-	@Autowired
-	@Qualifier("mySubstanceDaoDstu3")
-	private IFhirResourceDao<Substance> mySubstanceDao;
-	@Autowired
-	@Qualifier("mySystemDaoDstu3")
-	private IFhirSystemDao<Bundle, Meta> mySystemDao;
-	@Autowired
-	@Qualifier("mySystemProviderDstu3")
-	private JpaSystemProviderDstu3 mySystemProvider;
-	@Autowired
-	@Qualifier("myJpaValidationSupportChainDstu3")
-	private IValidationSupport myValidationSupport;
-	@Autowired
-	private IResourceReindexingSvc myResourceReindexingSvc;
-	@Autowired
-	private IBulkDataExportSvc myBulkDataExportSvc;
-
-	@Before
-	public void beforePurgeDatabase() {
-		purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc);
-	}
-
-	@Before
-	public void beforeResetConfig() {
-		myDaoConfig.setHardSearchLimit(1000);
-		myDaoConfig.setHardTagListLimit(1000);
-		myDaoConfig.setIncludeLimit(2000);
-	}
-
-	@Override
-	protected FhirContext getContext() {
-		return myFhirCtx;
-	}
-
-	@Override
-	protected PlatformTransactionManager getTxManager() {
-		return myTxManager;
-	}
-
-	@Test
-	public void testSearchWithContent() throws Exception {
-		String methodName = "testEverythingIncludesBackReferences";
-
-		Organization org = new Organization();
-		org.setName(methodName);
-		IIdType orgId = myOrganizationDao.create(org, mySrd).getId().toUnqualifiedVersionless();
-
-		SearchParameterMap map = new SearchParameterMap();
-		map.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, new StringParam(methodName));
-		try {
-			myOrganizationDao.search(map).size();
-			fail();
-		} catch (InvalidRequestException e) {
-			assertEquals("Fulltext search is not enabled on this service, can not process parameter: _content", e.getMessage());
-		}
-	}
-
-	@Test
-	public void testSearchWithRegularParam() throws Exception {
-		String methodName = "testEverythingIncludesBackReferences";
-
-		Organization org = new Organization();
-		org.setName(methodName);
-		IIdType orgId = myOrganizationDao.create(org, mySrd).getId().toUnqualifiedVersionless();
-
-		SearchParameterMap map = new SearchParameterMap();
-		map.add(Organization.SP_NAME, new StringParam(methodName));
-		myOrganizationDao.search(map);
-
-	}
-
-	@Test
-	public void testSearchWithText() throws Exception {
-		String methodName = "testEverythingIncludesBackReferences";
-
-		Organization org = new Organization();
-		org.setName(methodName);
-		IIdType orgId = myOrganizationDao.create(org, mySrd).getId().toUnqualifiedVersionless();
-
-		SearchParameterMap map = new SearchParameterMap();
-		map.add(ca.uhn.fhir.rest.api.Constants.PARAM_TEXT, new StringParam(methodName));
-		try {
-			myOrganizationDao.search(map).size();
-			fail();
-		} catch (InvalidRequestException e) {
-			assertEquals("Fulltext search is not enabled on this service, can not process parameter: _text", e.getMessage());
-		}
-	}
-
-
-	@AfterClass
-	public static void afterClassClearContext() {
-		TestUtil.clearAllStaticFieldsForUnitTest();
-	}
-
-}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchFtTest.java

@@ -1,25 +1,26 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
-import static org.hamcrest.Matchers.*;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
-
-import java.util.List;
-
-import javax.servlet.http.HttpServletRequest;
-
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
-import org.hl7.fhir.r4.model.*;
-import org.hl7.fhir.r4.model.Observation.ObservationStatus;
-import org.hl7.fhir.instance.model.api.IIdType;
-import org.junit.*;
-
 import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl.Suggestion;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.param.*;
 import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
 import ca.uhn.fhir.util.TestUtil;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.*;
+import org.hl7.fhir.r4.model.Observation.ObservationStatus;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.List;
+
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
 
 public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 
@@ -30,12 +31,6 @@ public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 		myDaoConfig.setReuseCachedSearchResultsForMillis(null);
 	}
 
-	@AfterClass
-	public static void afterClassClearContext() {
-		TestUtil.clearAllStaticFieldsForUnitTest();
-	}
-
-
 	@Test
 	public void testCodeTextSearch() {
 		Observation obs1 = new Observation();
@@ -72,7 +67,6 @@ public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 
 	}
 
-
 	@Test
 	public void testResourceTextSearch() {
 		Observation obs1 = new Observation();
@@ -127,7 +121,6 @@ public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 
 	}
 
-	
 	@Test
 	public void testSuggestIgnoresBase64Content() {
 		Patient patient = new Patient();
@@ -251,7 +244,6 @@ public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 
 	}
 
-	
 	@Test
 	public void testSearchAndReindex() {
 		Patient patient;
@@ -478,7 +470,6 @@ public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 
 	}
 
-	
 	/**
 	 * When processing transactions, we do two passes. Make sure we don't update the lucene index twice since that would
 	 * be inefficient
@@ -576,4 +567,9 @@ public class FhirResourceDaoR4SearchFtTest extends BaseJpaR4Test {
 
 	}
 
+	@AfterClass
+	public static void afterClassClearContext() {
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java

@@ -527,7 +527,12 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test {
 		/*
 		 * 20 should be prefetched since that's the initial page size
 		 */
-
+		await().until(()->{
+			return runInTransaction(()->{
+				Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException(""));
+				return search.getNumFound() == 20;
+			});
+		});
 		runInTransaction(() -> {
 			Search search = mySearchEntityDao.findByUuidAndFetchIncludes(uuid).orElseThrow(() -> new InternalErrorException(""));
 			assertEquals(20, search.getNumFound());

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchTest.java

@@ -0,0 +1,230 @@
+package ca.uhn.fhir.jpa.dao.r4;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc;
+import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticSearch;
+import ca.uhn.fhir.jpa.dao.*;
+import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
+import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
+import ca.uhn.fhir.jpa.entity.TermConcept;
+import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
+import ca.uhn.fhir.jpa.model.entity.ResourceTable;
+import ca.uhn.fhir.jpa.provider.r4.JpaSystemProviderR4;
+import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc;
+import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
+import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
+import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
+import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
+import ca.uhn.fhir.jpa.term.IHapiTerminologySvcR4;
+import ca.uhn.fhir.parser.IParser;
+import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.param.StringParam;
+import ca.uhn.fhir.util.TestUtil;
+import ca.uhn.fhir.validation.FhirValidator;
+import ca.uhn.fhir.validation.ValidationResult;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.hapi.ctx.IValidationSupport;
+import org.hl7.fhir.r4.model.*;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import javax.persistence.EntityManager;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = {TestR4ConfigWithElasticSearch.class})
+public class FhirResourceDaoR4SearchWithElasticSearchTest extends BaseJpaTest {
+	public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system";
+	public static final String URL_MY_VALUE_SET = "http://example.com/my_value_set";
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4SearchWithElasticSearchTest.class);
+	@Autowired
+	protected DaoConfig myDaoConfig;
+	@Autowired
+	protected PlatformTransactionManager myTxManager;
+	@Autowired
+	protected ISearchParamPresenceSvc mySearchParamPresenceSvc;
+	@Autowired
+	protected ISearchCoordinatorSvc mySearchCoordinatorSvc;
+	@Autowired
+	protected ISearchParamRegistry mySearchParamRegistry;
+	@Autowired
+	@Qualifier("myValueSetDaoR4")
+	protected IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao;
+	@Autowired
+	protected IHapiTerminologySvcR4 myTermSvc;
+	@Autowired
+	protected IResourceTableDao myResourceTableDao;
+	@Autowired
+	@Qualifier("myCodeSystemDaoR4")
+	private IFhirResourceDao<CodeSystem> myCodeSystemDao;
+	@Autowired
+	private FhirContext myFhirCtx;
+	@Autowired
+	@Qualifier("myObservationDaoR4")
+	private IFhirResourceDao<Observation> myObservationDao;
+	@Autowired
+	@Qualifier("mySystemDaoR4")
+	private IFhirSystemDao<Bundle, Meta> mySystemDao;
+	@Autowired
+	private IResourceReindexingSvc myResourceReindexingSvc;
+	@Autowired
+	private IBulkDataExportSvc myBulkDataExportSvc;
+
+	@Before
+	public void beforePurgeDatabase() {
+		purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc);
+	}
+
+	@Override
+	protected FhirContext getContext() {
+		return myFhirCtx;
+	}
+
+	@Override
+	protected PlatformTransactionManager getTxManager() {
+		return myTxManager;
+	}
+
+	@Test
+	public void testResourceTextSearch() throws InterruptedException {
+		Observation obs1 = new Observation();
+		obs1.getCode().setText("Systolic Blood Pressure");
+		obs1.setStatus(Observation.ObservationStatus.FINAL);
+		obs1.setValue(new Quantity(123));
+		obs1.getNoteFirstRep().setText("obs1");
+		IIdType id1 = myObservationDao.create(obs1, mySrd).getId().toUnqualifiedVersionless();
+
+		Observation obs2 = new Observation();
+		obs2.getCode().setText("Diastolic Blood Pressure");
+		obs2.setStatus(Observation.ObservationStatus.FINAL);
+		obs2.setValue(new Quantity(81));
+		IIdType id2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless();
+
+		SearchParameterMap map;
+
+		map = new SearchParameterMap();
+		map.add(ca.uhn.fhir.rest.api.Constants.PARAM_CONTENT, new StringParam("systolic"));
+		assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map)), containsInAnyOrder(toValues(id1)));
+
+		map = new SearchParameterMap();
+		map.add(Constants.PARAM_CONTENT, new StringParam("blood"));
+		assertThat(toUnqualifiedVersionlessIdValues(myObservationDao.search(map)), containsInAnyOrder(toValues(id1, id2)));
+
+	}
+
+	@Test
+	public void testExpandWithIsAInExternalValueSet() {
+		createExternalCsAndLocalVs();
+
+		ValueSet vs = new ValueSet();
+		ValueSet.ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem(URL_MY_CODE_SYSTEM);
+		include.addFilter().setOp(ValueSet.FilterOperator.ISA).setValue("childAA").setProperty("concept");
+
+		ValueSet result = myValueSetDao.expand(vs, null);
+		logAndValidateValueSet(result);
+
+		ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
+		assertThat(codes, containsInAnyOrder("childAAA", "childAAB"));
+
+
+	}
+
+	private CodeSystem createExternalCs() {
+		CodeSystem codeSystem = new CodeSystem();
+		codeSystem.setUrl(URL_MY_CODE_SYSTEM);
+		codeSystem.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
+		IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified();
+
+		ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new);
+
+		TermCodeSystemVersion cs = new TermCodeSystemVersion();
+		cs.setResource(table);
+
+		TermConcept parentA = new TermConcept(cs, "ParentA").setDisplay("Parent A");
+		cs.getConcepts().add(parentA);
+
+		TermConcept childAA = new TermConcept(cs, "childAA").setDisplay("Child AA");
+		parentA.addChild(childAA, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+
+		TermConcept childAAA = new TermConcept(cs, "childAAA").setDisplay("Child AAA");
+		childAA.addChild(childAAA, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+
+		TermConcept childAAB = new TermConcept(cs, "childAAB").setDisplay("Child AAB");
+		childAA.addChild(childAAB, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+
+		TermConcept childAB = new TermConcept(cs, "childAB").setDisplay("Child AB");
+		parentA.addChild(childAB, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+
+		TermConcept parentB = new TermConcept(cs, "ParentB").setDisplay("Parent B");
+		cs.getConcepts().add(parentB);
+
+		TermConcept childBA = new TermConcept(cs, "childBA").setDisplay("Child BA");
+		childBA.addChild(childAAB, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+		parentB.addChild(childBA, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+
+		TermConcept parentC = new TermConcept(cs, "ParentC").setDisplay("Parent C");
+		cs.getConcepts().add(parentC);
+
+		TermConcept childCA = new TermConcept(cs, "childCA").setDisplay("Child CA");
+		parentC.addChild(childCA, TermConceptParentChildLink.RelationshipTypeEnum.ISA);
+
+		myTermSvc.storeNewCodeSystemVersion(table.getId(), URL_MY_CODE_SYSTEM, "SYSTEM NAME", "SYSTEM VERSION", cs, table);
+		return codeSystem;
+	}
+
+	private void createExternalCsAndLocalVs() {
+		CodeSystem codeSystem = createExternalCs();
+
+		createLocalVs(codeSystem);
+	}
+
+	private void createLocalVs(CodeSystem codeSystem) {
+		ValueSet valueSet = new ValueSet();
+		valueSet.setUrl(URL_MY_VALUE_SET);
+		valueSet.getCompose().addInclude().setSystem(codeSystem.getUrl());
+		myValueSetDao.create(valueSet, mySrd);
+	}
+
+	private ArrayList<String> toCodesContains(List<ValueSet.ValueSetExpansionContainsComponent> theContains) {
+		ArrayList<String> retVal = new ArrayList<String>();
+		for (ValueSet.ValueSetExpansionContainsComponent next : theContains) {
+			retVal.add(next.getCode());
+		}
+		return retVal;
+	}
+
+
+	private void logAndValidateValueSet(ValueSet theResult) {
+		IParser parser = myFhirCtx.newXmlParser().setPrettyPrint(true);
+		String encoded = parser.encodeResourceToString(theResult);
+		ourLog.info(encoded);
+
+		FhirValidator validator = myFhirCtx.newValidator();
+		validator.setValidateAgainstStandardSchema(true);
+		validator.setValidateAgainstStandardSchematron(true);
+		ValidationResult result = validator.validateWithResult(theResult);
+
+		assertEquals(0, result.getMessages().size());
+
+	}
+
+
+	@AfterClass
+	public static void afterClassClearContext() {
+		TestUtil.clearAllStaticFieldsForUnitTest();
+	}
+
+}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithLuceneDisabledTest.java

@@ -1,9 +1,8 @@
 package ca.uhn.fhir.jpa.dao.r4;
 
 import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.jpa.config.TestR4Config;
 import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc;
-import ca.uhn.fhir.jpa.config.TestR4WithoutLuceneConfig;
+import ca.uhn.fhir.jpa.config.TestR4WithLuceneDisabledConfig;
 import ca.uhn.fhir.jpa.dao.*;
 import ca.uhn.fhir.jpa.search.ISearchCoordinatorSvc;
 import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
@@ -27,7 +26,6 @@ import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.annotation.Transactional;
@@ -40,7 +38,7 @@ import static org.junit.Assert.*;
 import static org.mockito.ArgumentMatchers.any;
 
 @RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes = {TestR4WithoutLuceneConfig.class})
+@ContextConfiguration(classes = {TestR4WithLuceneDisabledConfig.class})
 public class FhirResourceDaoR4SearchWithLuceneDisabledTest extends BaseJpaTest {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4SearchWithLuceneDisabledTest.class);
 	@Autowired

hapi-fhir-jpaserver-base/src/test/resources/logback-test.xml

@@ -1,7 +1,7 @@
 <configuration>
 	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 		<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-			<level>INFO</level>
+			<level>TRACE</level>
 		</filter>
 		<encoder>
 			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%file:%line] %msg%n</pattern>
@@ -40,6 +40,11 @@
 		<appender-ref ref="STDOUT" />
 	</logger>
 
+	<logger name="org.hibernate.search.elasticsearch.request" additivity="false" level="trace">
+		<appender-ref ref="STDOUT" />
+	</logger>
+
+
 	<root level="info">
 		<appender-ref ref="STDOUT" />
 	</root>

hapi-fhir-jpaserver-elasticsearch/pom.xml

@@ -1,23 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-
-	<parent>
-		<groupId>ca.uhn.hapi.fhir</groupId>
-		<artifactId>hapi-deployable-pom</artifactId>
-		<version>4.1.0-SNAPSHOT</version>
-		<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
-	</parent>
-
-	<artifactId>hapi-fhir-jpaserver-elasticsearch</artifactId>
-	<packaging>jar</packaging>
-
-	<name>HAPI FHIR JPA Server - ElasticSearch Integration</name>
-
-	<dependencies>
-		<dependency>
-			<groupId>org.hibernate</groupId>
-			<artifactId>hibernate-search-elasticsearch</artifactId>
-		</dependency>
-	</dependencies>
-
-</project>

hapi-fhir-jpaserver-example/src/main/java/ca/uhn/fhir/jpa/demo/elasticsearch/FhirServerConfig.java

@@ -6,7 +6,7 @@ import javax.sql.DataSource;
 
 import ca.uhn.fhir.jpa.config.BaseJavaConfigDstu3;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
-import ca.uhn.fhir.jpa.search.ElasticsearchMappingProvider;
+import ca.uhn.fhir.jpa.search.elastic.ElasticsearchMappingProvider;
 import ca.uhn.fhir.jpa.util.SubscriptionsRequireManualActivationInterceptorDstu3;
 import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor;
 import ca.uhn.fhir.rest.server.interceptor.ResponseHighlighterInterceptor;

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/OperationParameter.java

@@ -14,6 +14,7 @@ import ca.uhn.fhir.rest.api.ValidationModeEnum;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.param.BaseAndListParam;
 import ca.uhn.fhir.rest.param.DateRangeParam;
+import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.param.binder.CollectionBinder;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
@@ -21,14 +22,12 @@ import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
 import ca.uhn.fhir.util.FhirTerser;
 import ca.uhn.fhir.util.ReflectionUtil;
 import org.apache.commons.lang3.Validate;
-import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.*;
-import org.hl7.fhir.instance.model.api.IBaseDatatype;
-import org.hl7.fhir.instance.model.api.IBaseResource;
-import org.hl7.fhir.instance.model.api.IPrimitiveType;
 
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.*;
+import java.util.function.Consumer;
 
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
@@ -163,7 +162,10 @@ public class OperationParameter implements IParameter {
 		 */
 		isSearchParam &= typeIsConcrete && !IBase.class.isAssignableFrom(myParameterType);
 
-		myAllowGet = IPrimitiveType.class.isAssignableFrom(myParameterType) || String.class.equals(myParameterType) || isSearchParam || ValidationModeEnum.class.equals(myParameterType);
+		myAllowGet = IPrimitiveType.class.isAssignableFrom(myParameterType)
+			|| String.class.equals(myParameterType)
+			|| isSearchParam
+			|| ValidationModeEnum.class.equals(myParameterType);
 
 		/*
 		 * The parameter can be of type string for validation methods - This is a bit weird. See ValidateDstu2Test. We
@@ -172,6 +174,12 @@ public class OperationParameter implements IParameter {
 		if (!myParameterType.equals(IBase.class) && !myParameterType.equals(String.class)) {
 			if (IBaseResource.class.isAssignableFrom(myParameterType) && myParameterType.isInterface()) {
 				myParamType = "Resource";
+			} else if (IBaseReference.class.isAssignableFrom(myParameterType)) {
+				myParamType = "Reference";
+				myAllowGet = true;
+			} else if (IBaseCoding.class.isAssignableFrom(myParameterType)) {
+				myParamType = "Coding";
+				myAllowGet = true;
 			} else if (DateRangeParam.class.isAssignableFrom(myParameterType)) {
 				myParamType = "date";
 				myMax = 2;
@@ -266,7 +274,7 @@ public class OperationParameter implements IParameter {
 				if (myAllowGet) {
 
 					if (DateRangeParam.class.isAssignableFrom(myParameterType)) {
-						List<QualifiedParamList> parameters = new ArrayList<QualifiedParamList>();
+						List<QualifiedParamList> parameters = new ArrayList<>();
 						parameters.add(QualifiedParamList.singleton(paramValues[0]));
 						if (paramValues.length > 1) {
 							parameters.add(QualifiedParamList.singleton(paramValues[1]));
@@ -275,11 +283,31 @@ public class OperationParameter implements IParameter {
 						FhirContext ctx = theRequest.getServer().getFhirContext();
 						dateRangeParam.setValuesAsQueryTokens(ctx, myName, parameters);
 						matchingParamValues.add(dateRangeParam);
+
+					} else if (IBaseReference.class.isAssignableFrom(myParameterType)) {
+
+						processAllCommaSeparatedValues(paramValues, t -> {
+							IBaseReference param = (IBaseReference) ReflectionUtil.newInstance(myParameterType);
+							param.setReference(t);
+							matchingParamValues.add(param);
+						});
+
+					} else if (IBaseCoding.class.isAssignableFrom(myParameterType)) {
+
+						processAllCommaSeparatedValues(paramValues, t -> {
+							TokenParam tokenParam = new TokenParam();
+							tokenParam.setValueAsQueryToken(myContext, myName, null, t);
+
+							IBaseCoding param = (IBaseCoding) ReflectionUtil.newInstance(myParameterType);
+							param.setSystem(tokenParam.getSystem());
+							param.setCode(tokenParam.getValue());
+							matchingParamValues.add(param);
+						});
+
 					} else if (String.class.isAssignableFrom(myParameterType)) {
 
-						for (String next : paramValues) {
+						matchingParamValues.addAll(Arrays.asList(paramValues));
-							matchingParamValues.add(next);
+
-						}
 					} else if (ValidationModeEnum.class.equals(myParameterType)) {
 
 						if (isNotBlank(paramValues[0])) {
@@ -309,6 +337,22 @@ public class OperationParameter implements IParameter {
 		}
 	}
 
+	/**
+	 * This method is here to mediate between the POST form of operation parameters (i.e. elements within a <code>Parameters</code>
+	 * resource) and the GET form (i.e. URL parameters).
+	 * <p>
+	 * Essentially we want to allow comma-separated values as is done with searches on URLs.
+	 * </p>
+	 */
+	private void processAllCommaSeparatedValues(String[] theParamValues, Consumer<String> theHandler) {
+		for (String nextValue : theParamValues) {
+			QualifiedParamList qualifiedParamList = QualifiedParamList.splitQueryStringByCommasIgnoreEscape(null, nextValue);
+			for (String nextSplitValue : qualifiedParamList) {
+				theHandler.accept(nextSplitValue);
+			}
+		}
+	}
+
 	private void translateQueryParametersIntoServerArgumentForPost(RequestDetails theRequest, List<Object> matchingParamValues) {
 		IBaseResource requestContents = (IBaseResource) theRequest.getUserData().get(REQUEST_CONTENTS_USERDATA_KEY);
 		if (requestContents != null) {
@@ -394,10 +438,6 @@ public class OperationParameter implements IParameter {
 		}
 	}
 
-	public static void throwInvalidMode(String paramValues) {
-		throw new InvalidRequestException("Invalid mode value: \"" + paramValues + "\"");
-	}
-
 	interface IOperationParamConverter {
 
 		Object incomingServer(Object theObject);
@@ -429,5 +469,9 @@ public class OperationParameter implements IParameter {
 
 	}
 
+	public static void throwInvalidMode(String paramValues) {
+		throw new InvalidRequestException("Invalid mode value: \"" + paramValues + "\"");
+	}
+
 
 }

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/BaseLastNProvider.java

@@ -0,0 +1,43 @@
+package ca.uhn.fhir.rest.server.provider;
+
+import ca.uhn.fhir.rest.annotation.Operation;
+import ca.uhn.fhir.rest.annotation.OperationParam;
+import ca.uhn.fhir.rest.api.Constants;
+import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
+import org.hl7.fhir.instance.model.api.IBaseBundle;
+import org.hl7.fhir.instance.model.api.IBaseCoding;
+import org.hl7.fhir.instance.model.api.IBaseReference;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
+
+import java.util.List;
+
+/**
+ * This class implements the Observation
+ * <a href="http://hl7.org/fhir/observation-operation-lastn.html">$lastn</a> operation.
+ * <p>
+ * It is does not implement the actual storage logic for this operation, but can be
+ * subclassed to provide this functionality.
+ * </p>
+ *
+ * @since 4.1.0
+ */
+public abstract class BaseLastNProvider {
+
+	@Operation(name = Constants.OPERATION_LASTN, typeName = "Observation", idempotent = true)
+	public IBaseBundle lastN(
+		ServletRequestDetails theRequestDetails,
+		@OperationParam(name = "subject", typeName = "reference", min = 0, max = 1) IBaseReference theSubject,
+		@OperationParam(name = "category", typeName = "coding", min = 0, max = OperationParam.MAX_UNLIMITED) List<IBaseCoding> theCategories,
+		@OperationParam(name = "code", typeName = "coding", min = 0, max = OperationParam.MAX_UNLIMITED) List<IBaseCoding> theCodes,
+		@OperationParam(name = "max", typeName = "integer", min = 0, max = 1) IPrimitiveType<Integer> theMax
+	) {
+		return processLastN(theSubject, theCategories, theCodes, theMax);
+	}
+
+	/**
+	 * Subclasses should implement this method
+	 */
+	protected abstract IBaseBundle processLastN(IBaseReference theSubject, List<IBaseCoding> theCategories, List<IBaseCoding> theCodes, IPrimitiveType<Integer> theMax);
+
+
+}

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/BaseR4ServerTest.java

@@ -0,0 +1,44 @@
+package ca.uhn.fhir.rest.server;
+
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.context.api.BundleInclusionRule;
+import ca.uhn.fhir.rest.api.EncodingEnum;
+import ca.uhn.fhir.rest.client.api.IGenericClient;
+import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum;
+import ca.uhn.fhir.test.utilities.JettyUtil;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.junit.After;
+
+public class BaseR4ServerTest {
+	private FhirContext myCtx = FhirContext.forR4();
+	private Server myServer;
+	protected IGenericClient myClient;
+	protected String myBaseUrl;
+
+	@After
+	public void after() throws Exception {
+		JettyUtil.closeServer(myServer);
+	}
+
+	protected void startServer(Object theProvider) throws Exception {
+		RestfulServer servlet = new RestfulServer(myCtx);
+		servlet.registerProvider(theProvider);
+		ServletHandler proxyHandler = new ServletHandler();
+		servlet.setDefaultResponseEncoding(EncodingEnum.XML);
+		servlet.setBundleInclusionRule(BundleInclusionRule.BASED_ON_RESOURCE_PRESENCE);
+		ServletHolder servletHolder = new ServletHolder(servlet);
+		proxyHandler.addServletWithMapping(servletHolder, "/*");
+
+		myServer = new Server(0);
+		myServer.setHandler(proxyHandler);
+		JettyUtil.startServer(myServer);
+		int port = JettyUtil.getPortForStartedServer(myServer);
+
+		myBaseUrl = "http://localhost:" + port;
+		myCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
+		myClient = myCtx.newRestfulGenericClient(myBaseUrl);
+	}
+
+}

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/LastNProviderTest.java

@@ -0,0 +1,64 @@
+package ca.uhn.fhir.rest.server;
+
+import ca.uhn.fhir.rest.server.provider.BaseLastNProvider;
+import org.hl7.fhir.instance.model.api.IBaseBundle;
+import org.hl7.fhir.instance.model.api.IBaseCoding;
+import org.hl7.fhir.instance.model.api.IBaseReference;
+import org.hl7.fhir.instance.model.api.IPrimitiveType;
+import org.hl7.fhir.r4.model.Bundle;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class LastNProviderTest extends BaseR4ServerTest {
+
+	private IBaseReference myLastSubject;
+	private List<IBaseCoding> myLastCategories;
+	private List<IBaseCoding> myLastCodes;
+	private IPrimitiveType<Integer> myLastMax;
+
+	@Test
+	public void testAllParamsPopulated() throws Exception {
+
+		class MyProvider extends BaseLastNProvider {
+
+			@Override
+			protected IBaseBundle processLastN(IBaseReference theSubject, List<IBaseCoding> theCategories, List<IBaseCoding> theCodes, IPrimitiveType<Integer> theMax) {
+				myLastSubject = theSubject;
+				myLastCategories = theCategories;
+				myLastCodes = theCodes;
+				myLastMax = theMax;
+
+				Bundle retVal = new Bundle();
+				retVal.setId("abc123");
+				retVal.setType(Bundle.BundleType.SEARCHSET);
+				return retVal;
+			}
+		}
+		MyProvider provider = new MyProvider();
+		startServer(provider);
+
+		Bundle response = myClient
+			.search()
+			.byUrl(myBaseUrl + "/Observation/$lastn?subject=Patient/123&category=http://terminology.hl7.org/CodeSystem/observation-category|laboratory,http://terminology.hl7.org/CodeSystem/observation-category|vital-signs&code=http://loinc.org|1111-1,http://loinc.org|2222-2&max=15")
+			.returnBundle(Bundle.class)
+			.execute();
+		assertEquals("abc123", response.getIdElement().getIdPart());
+		assertEquals("Patient/123", myLastSubject.getReferenceElement().getValue());
+		assertEquals(2, myLastCategories.size());
+		assertEquals("http://terminology.hl7.org/CodeSystem/observation-category", myLastCategories.get(0).getSystem());
+		assertEquals("laboratory", myLastCategories.get(0).getCode());
+		assertEquals("http://terminology.hl7.org/CodeSystem/observation-category", myLastCategories.get(1).getSystem());
+		assertEquals("vital-signs", myLastCategories.get(1).getCode());
+		assertEquals(2, myLastCodes.size());
+		assertEquals("http://loinc.org", myLastCodes.get(0).getSystem());
+		assertEquals("1111-1", myLastCodes.get(0).getCode());
+		assertEquals("http://loinc.org", myLastCodes.get(1).getSystem());
+		assertEquals("2222-2", myLastCodes.get(1).getCode());
+		assertEquals(15, myLastMax.getValue().intValue());
+	}
+
+
+}

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/ServerMethodSelectionR4Test.java

@@ -1,24 +1,15 @@
 package ca.uhn.fhir.rest.server;
 
-import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.context.api.BundleInclusionRule;
 import ca.uhn.fhir.model.api.Include;
 import ca.uhn.fhir.rest.annotation.IncludeParam;
 import ca.uhn.fhir.rest.annotation.OptionalParam;
 import ca.uhn.fhir.rest.annotation.Search;
-import ca.uhn.fhir.rest.api.EncodingEnum;
-import ca.uhn.fhir.rest.client.api.IGenericClient;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
-import ca.uhn.fhir.test.utilities.JettyUtil;
 import com.google.common.collect.Lists;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.Patient;
 import org.hl7.fhir.r4.model.StringType;
-import org.junit.After;
 import org.junit.Test;
 
 import java.util.List;
@@ -27,18 +18,9 @@ import java.util.Set;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.junit.Assert.*;
 
-public class ServerMethodSelectionR4Test {
+public class ServerMethodSelectionR4Test extends BaseR4ServerTest {
 
 
-	private FhirContext myCtx = FhirContext.forR4();
-	private Server myServer;
-	private IGenericClient myClient;
-
-	@After
-	public void after() throws Exception {
-		JettyUtil.closeServer(myServer);
-	}
-
 	/**
 	 * Server method with no _include
 	 * Client request with _include
@@ -161,22 +143,6 @@ public class ServerMethodSelectionR4Test {
 		assertEquals(1, results.getEntry().size());
 	}
 
-	private void startServer(Object theProvider) throws Exception {
-		RestfulServer servlet = new RestfulServer(myCtx);
-		servlet.registerProvider(theProvider);
-		ServletHandler proxyHandler = new ServletHandler();
-		servlet.setDefaultResponseEncoding(EncodingEnum.XML);
-		servlet.setBundleInclusionRule(BundleInclusionRule.BASED_ON_RESOURCE_PRESENCE);
-		ServletHolder servletHolder = new ServletHolder(servlet);
-		proxyHandler.addServletWithMapping(servletHolder, "/*");
-
-		myServer = new Server(0);
-		myServer.setHandler(proxyHandler);
-		JettyUtil.startServer(myServer);
-		int port = JettyUtil.getPortForStartedServer(myServer);
-
-		myClient = myCtx.newRestfulGenericClient("http://localhost:" + port);
-	}
 
 
 	public static class MyBaseProvider implements IResourceProvider {
@@ -185,6 +151,7 @@ public class ServerMethodSelectionR4Test {
 		public Class<? extends IBaseResource> getResourceType() {
 			return Patient.class;
 		}
+
 	}
 
 }

pom.xml

@@ -601,7 +601,7 @@
 		<!--<hibernate_version>5.2.10.Final</hibernate_version>-->
 		<hibernate_version>5.4.4.Final</hibernate_version>
 		<!-- Update lucene version when you update hibernate-search version -->
-		<hibernate_search_version>5.11.1.Final</hibernate_search_version>
+		<hibernate_search_version>5.11.3.Final</hibernate_search_version>
 		<lucene_version>5.5.5</lucene_version>
 		<hibernate_validator_version>5.4.2.Final</hibernate_validator_version>
 		<httpcore_version>4.4.11</httpcore_version>
@@ -857,18 +857,6 @@
 				<!--<version>6.2.2.jre8</version>-->
 				<version>7.0.0.jre8</version>
 			</dependency>
-			<!--
-			<dependency>
-				<groupId>com.sun.xml.bind</groupId>
-				<artifactId>jaxb-core</artifactId>
-				<version>${jaxb_core_version}</version>
-			</dependency>
-			<dependency>
-				<groupId>com.sun.xml.bind</groupId>
-				<artifactId>jaxb-impl</artifactId>
-				<version>${jaxb_core_version}</version>
-			</dependency>
-			-->
 			<dependency>
 				<groupId>javax.mail</groupId>
 				<artifactId>javax.mail-api</artifactId>
@@ -1405,6 +1393,11 @@
 				<artifactId>xmlunit-core</artifactId>
 				<version>2.4.0</version>
 			</dependency>
+			<dependency>
+				<groupId>pl.allegro.tech</groupId>
+				<artifactId>embedded-elasticsearch</artifactId>
+				<version>2.10.0</version>
+			</dependency>
 			<dependency>
 				<groupId>xpp3</groupId>
 				<artifactId>xpp3</artifactId>
@@ -1669,6 +1662,11 @@
 					<groupId>org.jacoco</groupId>
 					<artifactId>jacoco-maven-plugin</artifactId>
 					<version>0.8.4</version>
+					<configuration>
+						<excludes>
+							<exclude>ca/uhn/fhir/model/dstu2/**/*.class</exclude>
+						</excludes>
+					</configuration>
 				</plugin>
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
@@ -2391,7 +2389,6 @@
 				<module>hapi-fhir-jaxrsserver-base</module>
 				<module>hapi-fhir-jaxrsserver-example</module>
 				<module>hapi-fhir-jpaserver-base</module>
-				<module>hapi-fhir-jpaserver-elasticsearch</module>
 				<module>hapi-fhir-jpaserver-migrate</module>
 				<module>restful-server-example</module>
 				<module>hapi-fhir-testpage-overlay</module>

src/changes/changes.xml

@@ -12,7 +12,8 @@
 				latest versions (dependent HAPI modules listed in brackets):
 				<![CDATA[
 				<ul>
-					<li>Hibernate Core (Core): 5.4.2.Final -&gt; 5.4.4.Final</li>
+					<li>Hibernate Core (JPA): 5.4.2.Final -&gt; 5.4.4.Final</li>
+					<li>Hibernate Search (JPA): 5.11.1.Final -&gt; 5.11.3.Final</li>
 					<li>Jackson Databind (JPA): 2.9.9 -&gt; 2.9.10 (CVE-2019-16335, CVE-2019-14540)</li>
 				</ul>
 				]]>
@@ -222,6 +223,11 @@
 				The JPA server failed to find codes defined in not-present codesystems in some cases, and reported
 				that the CodeSystem did not exist. This has been corrected.
 			</action>
+			<action type="add">
+				Support for ElasticSearch has been added to the JPA server directly (i.e. without needing a separate
+				module) and a new class called "ElasticsearchHibernatePropertiesBuilder" has been added to facilitate
+				the creation of relevant properties.
+			</action>
 		</release>
 		<release version="4.0.3" date="2019-09-03" description="Igloo (Point Release)">
 			<action type="fix">