More work on fulltext search and add a failing test for subscriptions

This commit is contained in:
jamesagnew 2015-10-09 09:12:41 -04:00
parent ad868038a8
commit b4c86d033e
21 changed files with 2273 additions and 2099 deletions

1
.gitignore vendored
View File

@ -14,6 +14,7 @@ nohup.out
.DS_Store
*.orig
tmp.txt
*.hprof
# Vagrant stuff.
.vagrant

View File

@ -2193,6 +2193,11 @@ public abstract class BaseHapiFhirResourceDao<T extends IResource> extends BaseH
List<Long> searchResultPids;
if (mySearchDao == null) {
if (theParams.containsKey(Constants.PARAM_TEXT)) {
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
} else if (theParams.containsKey(Constants.PARAM_CONTENT)) {
throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
}
searchResultPids = null;
} else {
searchResultPids = mySearchDao.search(getResourceName(), theParams);

View File

@ -92,6 +92,9 @@ public abstract class BaseHapiFhirSystemDao<T> extends BaseHapiFhirDao<IBaseReso
q.setMaxResults(maxResult);
List<ResourceTable> resources = q.getResultList();
if (resources.isEmpty()) {
return 0;
}
ourLog.info("Indexing {} resources", resources.size());

View File

@ -0,0 +1,28 @@
package ca.uhn.fhir.jpa.dao;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import ca.uhn.fhir.model.dstu2.resource.Bundle;
import ca.uhn.fhir.model.dstu2.resource.SearchParameter;
public class FhirResourceDaoSearchParameterDstu2 extends FhirResourceDaoDstu2<SearchParameter>implements IFhirResourceDaoSearchParameter<SearchParameter> {
@Autowired
private IFhirSystemDao<Bundle> mySystemDao;
/**
* This method is called once per minute to perform any required re-indexing. During most passes this will
* just check and find that there are no resources requiring re-indexing. In that case the method just returns
* immediately. If the search finds that some resources require reindexing, the system will do a bunch of
* reindexing and then return.
*/
@Scheduled(fixedDelay=DateUtils.MILLIS_PER_MINUTE)
public void performReindexingPass() {
mySystemDao.performReindexingPass(250);
}
}

View File

@ -92,10 +92,15 @@ public class FhirResourceDaoSubscriptionDstu2 extends FhirResourceDaoDstu2<Subsc
@Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public synchronized void pollForNewUndeliveredResourcesScheduler() {
pollForNewUndeliveredResources();
}
@Override
public synchronized void pollForNewUndeliveredResources() {
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public synchronized int pollForNewUndeliveredResources() {
if (getConfig().isSubscriptionEnabled() == false) {
return;
return 0;
}
ourLog.trace("Beginning pollForNewUndeliveredResources()");
@ -108,18 +113,21 @@ public class FhirResourceDaoSubscriptionDstu2 extends FhirResourceDaoDstu2<Subsc
TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
int retVal = 0;
for (final SubscriptionTable nextSubscriptionTable : subscriptions) {
txTemplate.execute(new TransactionCallback<Void>() {
retVal += txTemplate.execute(new TransactionCallback<Integer>() {
@Override
public Void doInTransaction(TransactionStatus theStatus) {
pollForNewUndeliveredResources(nextSubscriptionTable);
return null;
public Integer doInTransaction(TransactionStatus theStatus) {
return pollForNewUndeliveredResources(nextSubscriptionTable);
}
});
}
return retVal;
}
private void pollForNewUndeliveredResources(SubscriptionTable theSubscriptionTable) {
private int pollForNewUndeliveredResources(SubscriptionTable theSubscriptionTable) {
Subscription subscription = toResource(Subscription.class, theSubscriptionTable.getSubscriptionResource(), false);
RuntimeResourceDefinition resourceDef = validateCriteriaAndReturnResourceDefinition(subscription);
SearchParameterMap criteriaUrl = translateMatchUrl(subscription.getCriteria(), resourceDef);
@ -129,7 +137,7 @@ public class FhirResourceDaoSubscriptionDstu2 extends FhirResourceDaoDstu2<Subsc
long end = System.currentTimeMillis() - getConfig().getSubscriptionPollDelay();
if (end <= start) {
ourLog.trace("Skipping search for subscription");
return;
return 0;
}
ourLog.debug("Subscription {} search from {} to {}", new Object[] { subscription.getId().getIdPart(), new InstantDt(new Date(start)), new InstantDt(new Date(end)) });
@ -142,7 +150,7 @@ public class FhirResourceDaoSubscriptionDstu2 extends FhirResourceDaoDstu2<Subsc
IFhirResourceDao<? extends IBaseResource> dao = getDao(resourceDef.getImplementingClass());
IBundleProvider results = dao.search(criteriaUrl);
if (results.size() == 0) {
return;
return 0;
}
ourLog.info("Found {} new results for Subscription {}", results.size(), subscription.getId().getIdPart());
@ -152,8 +160,14 @@ public class FhirResourceDaoSubscriptionDstu2 extends FhirResourceDaoDstu2<Subsc
for (IBaseResource next : results.getResources(0, results.size())) {
Date updated = ResourceMetadataKeyEnum.PUBLISHED.get((IResource) next).getValue();
if (mostRecentMatch == null || mostRecentMatch.getTime() < updated.getTime()) {
if (mostRecentMatch == null) {
mostRecentMatch = updated;
} else {
long mostRecentMatchTime = mostRecentMatch.getTime();
long updatedTime = updated.getTime();
if (mostRecentMatchTime < updatedTime) {
mostRecentMatch = updated;
}
}
SubscriptionFlaggedResource nextFlag = new SubscriptionFlaggedResource();
@ -171,6 +185,8 @@ public class FhirResourceDaoSubscriptionDstu2 extends FhirResourceDaoDstu2<Subsc
theSubscriptionTable.setMostRecentMatch(mostRecentMatch);
myEntityManager.merge(theSubscriptionTable);
return results.size();
}
@Override

View File

@ -0,0 +1,27 @@
package ca.uhn.fhir.jpa.dao;
/*
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2015 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.hl7.fhir.instance.model.api.IBaseResource;
public interface IFhirResourceDaoSearchParameter<T extends IBaseResource> extends IFhirResourceDao<T> {
// nothing yet..
}

View File

@ -27,7 +27,7 @@ import org.hl7.fhir.instance.model.api.IIdType;
public interface IFhirResourceDaoSubscription<T extends IBaseResource> extends IFhirResourceDao<T> {
void pollForNewUndeliveredResources();
int pollForNewUndeliveredResources();
List<IBaseResource> getUndeliveredResourcesAndPurge(Long theSubscriptionPid);

View File

@ -86,7 +86,7 @@ public class ResourceTable extends BaseHasResource implements Serializable {
/**
* Holds the narrative text only - Used for Fulltext searching but not directly stored in the DB
*/
@Column(name = "SP_NARRATIVE_TEXT")
@Column(name = "SP_NARRATIVE_TEXT", length = Integer.MAX_VALUE - 1)
@Lob
@Field()
private String myNarrativeText;

View File

@ -40,7 +40,6 @@ import ca.uhn.fhir.rest.annotation.Operation;
import ca.uhn.fhir.rest.annotation.OperationParam;
import ca.uhn.fhir.rest.annotation.Transaction;
import ca.uhn.fhir.rest.annotation.TransactionParam;
import ca.uhn.fhir.rest.param.NumberParam;
public class JpaSystemProviderDstu2 extends BaseJpaSystemProvider<Bundle> {
@ -171,20 +170,6 @@ public class JpaSystemProviderDstu2 extends BaseJpaSystemProvider<Bundle> {
return retVal;
}
//@formatter:off
@Operation(name="$perform-reindexing-pass", idempotent=true, returnParameters= {
@OperationParam(name="count", type=IntegerDt.class)
})
//@formatter:on
public Parameters performReindexingPass(@OperationParam(min=0, max=1, name="count") IntegerDt theCount) {
Integer countIn = theCount != null && theCount.getValue()!= null ? theCount.getValue().intValue() : null;
int count = mySystemDao.performReindexingPass(countIn);
Parameters retVal = new Parameters();
retVal.addParameter().setName("count").setValue(new IntegerDt(count));
return retVal;
}
//@formatter:off
@Operation(name="$meta", idempotent=true, returnParameters= {
@OperationParam(name="return", type=MetaDt.class)

View File

@ -18,38 +18,13 @@ import ca.uhn.fhir.rest.server.IBundleProvider;
public class BaseJpaTest {
public static String loadClasspath(String resource) throws IOException {
InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream(resource);
if (bundleRes == null) {
throw new NullPointerException("Can not load " + resource);
}
String bundleStr = IOUtils.toString(bundleRes);
return bundleStr;
}
@AfterClass
public static void afterClassShutdownDerby() throws SQLException {
// try {
// DriverManager.getConnection("jdbc:derby:memory:myUnitTestDB;drop=true");
// } catch (SQLNonTransientConnectionException e) {
// // expected.. for some reason....
// }
}
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseJpaTest.class);
@SuppressWarnings({ "rawtypes" })
protected List toList(IBundleProvider theSearch) {
return theSearch.getResources(0, theSearch.size());
}
protected List<IIdType> toUnqualifiedVersionlessIds(IBundleProvider theFound) {
List<IIdType> retVal = new ArrayList<IIdType>();
List<IBaseResource> resources = theFound.getResources(0, theFound.size());
for (IBaseResource next : resources) {
retVal.add((IIdType) next.getIdElement().toUnqualifiedVersionless());
}
return retVal;
}
protected List<IIdType> toUnqualifiedVersionlessIds(Bundle theFound) {
List<IIdType> retVal = new ArrayList<IIdType>();
for (Entry next : theFound.getEntry()) {
@ -60,6 +35,17 @@ public class BaseJpaTest {
return retVal;
}
protected List<IIdType> toUnqualifiedVersionlessIds(IBundleProvider theFound) {
List<IIdType> retVal = new ArrayList<IIdType>();
int size = theFound.size();
ourLog.info("Found {} results", size);
List<IBaseResource> resources = theFound.getResources(0, size);
for (IBaseResource next : resources) {
retVal.add((IIdType) next.getIdElement().toUnqualifiedVersionless());
}
return retVal;
}
protected List<IIdType> toUnqualifiedVersionlessIds(List<IBaseResource> theFound) {
List<IIdType> retVal = new ArrayList<IIdType>();
for (IBaseResource next : theFound) {
@ -68,5 +54,22 @@ public class BaseJpaTest {
return retVal;
}
@AfterClass
public static void afterClassShutdownDerby() throws SQLException {
// try {
// DriverManager.getConnection("jdbc:derby:memory:myUnitTestDB;drop=true");
// } catch (SQLNonTransientConnectionException e) {
// // expected.. for some reason....
// }
}
public static String loadClasspath(String resource) throws IOException {
InputStream bundleRes = SystemProviderDstu2Test.class.getResourceAsStream(resource);
if (bundleRes == null) {
throw new NullPointerException("Can not load " + resource);
}
String bundleStr = IOUtils.toString(bundleRes);
return bundleStr;
}
}

View File

@ -1,10 +1,13 @@
package ca.uhn.fhir.jpa.dao;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import java.util.List;
import org.hamcrest.core.StringContains;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.AfterClass;
@ -22,6 +25,8 @@ import ca.uhn.fhir.model.dstu.resource.Observation;
import ca.uhn.fhir.model.dstu.resource.Organization;
import ca.uhn.fhir.model.dstu.resource.Patient;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.StringDt;
import ca.uhn.fhir.rest.server.Constants;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;

View File

@ -339,7 +339,9 @@ public class FhirResourceDaoDstu2SubscriptionTest extends BaseJpaDstu2Test {
List<IBaseResource> results;
List<IIdType> resultIds;
mySubscriptionDao.pollForNewUndeliveredResources();
assertEquals(4, mySubscriptionDao.pollForNewUndeliveredResources());
assertEquals(0, mySubscriptionDao.pollForNewUndeliveredResources());
results = mySubscriptionDao.getUndeliveredResourcesAndPurge(subsId1);
resultIds = toUnqualifiedVersionlessIds(results);
assertThat(resultIds, contains(afterId1, afterId2));
@ -364,6 +366,12 @@ public class FhirResourceDaoDstu2SubscriptionTest extends BaseJpaDstu2Test {
resultIds = toUnqualifiedVersionlessIds(results);
assertThat(resultIds, empty());
mySystemDao.markAllResourcesForReindexing();
mySystemDao.performReindexingPass(100);
assertEquals(6, mySubscriptionDao.pollForNewUndeliveredResources());
assertEquals(0, mySubscriptionDao.pollForNewUndeliveredResources());
}
@ -395,6 +403,12 @@ public class FhirResourceDaoDstu2SubscriptionTest extends BaseJpaDstu2Test {
assertNull(mySubscriptionTableDao.findOne(subsId1).getLastClientPoll());
assertEquals(0, mySubscriptionDao.pollForNewUndeliveredResources());
mySystemDao.markAllResourcesForReindexing();
mySystemDao.performReindexingPass(100);
assertEquals(1, mySubscriptionDao.pollForNewUndeliveredResources());
assertEquals(0, mySubscriptionDao.pollForNewUndeliveredResources());
Thread.sleep(100);
ourLog.info("Before: {}", System.currentTimeMillis());

View File

@ -36,8 +36,8 @@
<property name="hibernate.cache.use_second_level_cache" value="false" />
<property name="hibernate.cache.use_structured_entries" value="false" />
<property name="hibernate.search.default.directory_provider" value="filesystem"/>
<property name="hibernate.search.default.indexBase" value="./lucene_indexes"/>
<property name="hibernate.search.default.indexBase" value="./target/lucene_indexes"/>
<entry key="hibernate.search.lucene_version" value="LUCENE_CURRENT" />
<!--
<property name="hibernate.ejb.naming_strategy" value="ca.uhn.fhir.jpa.util.CustomNamingStrategy" />
-->

View File

@ -2,7 +2,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
<level>TRACE</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
@ -37,6 +37,10 @@
<logger name="org.hibernate.SQL" additivity="false" level="trace">
<appender-ref ref="STDOUT" />
</logger>
<!-- Set to 'trace' to enable SQL Value logging -->
<logger name="org.hibernate.type" additivity="false" level="info">
<appender-ref ref="STDOUT" />
</logger>
<root level="info">
<appender-ref ref="STDOUT" />

View File

@ -24,6 +24,38 @@
<property name="password" value="SA"/>
</bean>
<bean depends-on="dbServer" id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="myPersistenceDataSource" />
<!--
<property name="persistenceXmlLocation" value="classpath:META-INF/fhirtest_persistence.xml" />
-->
<property name="packagesToScan">
<list>
<value>ca.uhn.fhir.jpa.entity</value>
</list>
</property>
<property name="persistenceUnitName" value="FHIR_DSTU2" />
<property name="jpaVendorAdapter">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="showSql" value="false" />
<property name="generateDdl" value="true" />
<property name="databasePlatform" value="org.hibernate.dialect.DerbyTenSevenDialect" />
</bean>
</property>
<property name="jpaPropertyMap">
<map>
<entry key="hibernate.dialect" value="ca.uhn.fhir.jpa.util.HapiDerbyTenSevenDialect" />
<entry key="hibernate.hbm2ddl.auto" value="update" />
<entry key="hibernate.jdbc.batch_size" value="20" />
<entry key="hibernate.cache.use_minimal_puts" value="true" />
<entry key="hibernate.show_sql" value="false" />
<entry key="hibernate.cache.use_query_cache" value="false" />
<entry key="hibernate.cache.use_second_level_cache" value="false" />
<entry key="hibernate.cache.use_structured_entries" value="false" />
</map>
</property>
</bean>
<!--for mysql-->
<!--
<bean depends-on="dbServer" id="myPersistenceDataSource" class="org.apache.commons.dbcp2.BasicDataSource" destroy-method="close">
@ -36,21 +68,6 @@
</bean>
-->
<bean depends-on="dbServer" id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="myPersistenceDataSource" />
<property name="persistenceXmlLocation" value="classpath:META-INF/fhirtest_persistence.xml" />
<property name="persistenceUnitName" value="FHIR_UT" />
<property name="jpaVendorAdapter">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="showSql" value="false" />
<property name="generateDdl" value="true" />
<property name="databasePlatform" value="ca.uhn.fhir.jpa.util.HapiDerbyTenSevenDialect" />
<!-- <property name="databasePlatform" value="org.hibernate.dialect.HSQLDialect" />-->
<!-- <property name="databasePlatform" value="org.hibernate.dialect.MySQL5Dialect" /> -->
</bean>
</property>
</bean>
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
<property name="entityManagerFactory" ref="entityManagerFactory" />
</bean>

View File

@ -25,17 +25,39 @@
<bean depends-on="dbServer" id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="myPersistenceDataSource" />
<!--
<property name="persistenceXmlLocation" value="classpath:META-INF/fhirtest_persistence.xml" />
<property name="persistenceUnitName" value="FHIR_UT" />
-->
<property name="persistenceUnitName" value="FHIR_DSTU2" />
<property name="packagesToScan">
<list>
<value>ca.uhn.fhir.jpa.entity</value>
</list>
</property>
<property name="jpaVendorAdapter">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="showSql" value="false" />
<property name="generateDdl" value="true" />
<property name="databasePlatform" value="ca.uhn.fhir.jpa.util.HapiDerbyTenSevenDialect" />
<property name="databasePlatform" value="org.hibernate.dialect.DerbyTenSevenDialect" />
<!-- <property name="databasePlatform" value="ca.uhn.fhir.jpa.util.HapiDerbyTenSevenDialect" />-->
<!-- <property name="databasePlatform" value="org.hibernate.dialect.HSQLDialect" /> -->
<!-- <property name="databasePlatform" value="org.hibernate.dialect.MySQL5Dialect" /> -->
</bean>
</property>
<property name="jpaPropertyMap">
<map>
<entry key="hibernate.dialect" value="ca.uhn.fhir.jpa.util.HapiDerbyTenSevenDialect" />
<entry key="hibernate.hbm2ddl.auto" value="update" />
<entry key="hibernate.jdbc.batch_size" value="20" />
<entry key="hibernate.cache.use_minimal_puts" value="true" />
<entry key="hibernate.show_sql" value="false" />
<entry key="hibernate.cache.use_query_cache" value="false" />
<entry key="hibernate.cache.use_second_level_cache" value="false" />
<entry key="hibernate.cache.use_structured_entries" value="false" />
<entry key="hibernate.search.default.directory_provider" value="filesystem" />
<entry key="hibernate.search.default.indexBase" value="#{systemProperties['fhir.lucene.location.dstu2']};" />
</map>
</property>
</bean>
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">

View File

@ -29,6 +29,7 @@ public class UhnFhirTestApp {
// new File("target/testdb").mkdirs();
System.setProperty("fhir.db.location", "./target/testdb");
System.setProperty("fhir.db.location.dstu2", "./target/testdb_dstu2");
System.setProperty("fhir.lucene.location.dstu2", "./target/testlucene_dstu2");
System.setProperty("fhir.baseurl.dstu1", base + "Dstu1");
System.setProperty("fhir.baseurl.dstu2", base + "Dstu1");

View File

@ -44,7 +44,7 @@
#foreach ( $res in $resources )
<bean id="my${res.name}Dao${versionCapitalized}"
## Some resource types have customized DAOs for resource specific functionality
#if ( ${versionCapitalized} == 'Dstu2' && ( ${res.name} == 'Bundle' || ${res.name} == 'Encounter' || ${res.name} == 'Everything' || ${res.name} == 'Patient' || ${res.name} == 'Subscription' || ${res.name} == 'QuestionnaireResponse' || ${res.name} == 'ValueSet'))
#if ( ${versionCapitalized} == 'Dstu2' && ( ${res.name} == 'Bundle' || ${res.name} == 'Encounter' || ${res.name} == 'Everything' || ${res.name} == 'Patient' || ${res.name} == 'Subscription' || ${res.name} == 'ValueSet' || ${res.name} == 'QuestionnaireResponse' || ${res.name} == 'SearchParameter'))
class="ca.uhn.fhir.jpa.dao.FhirResourceDao${res.name}${versionCapitalized}">
#else
class="ca.uhn.fhir.jpa.dao.FhirResourceDao${versionCapitalized}">

View File

@ -219,7 +219,7 @@
<!-- Note on Hibernate versions: Hibernate 4.3+ uses JPA 2.1, which is too new for a number of platforms including JBoss EAP 6.x and Glassfish 3.0. Upgrade this
version with caution! Also note that if you change this, you may get a failure in hibernate4-maven-plugin. See the note in hapi-fhir-jpaserver-base/pom.xml's configuration
for that plugin... -->
<hibernate_version>5.0.1.Final</hibernate_version>
<hibernate_version>5.0.2.Final</hibernate_version>
<hibernate_validator_version>5.2.1.Final</hibernate_validator_version>
<jetty_version>9.2.6.v20141205</jetty_version>
<maven_build_helper_plugin_version>1.9.1</maven_build_helper_plugin_version>