Add ability to disable stale search expiry

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java

@@ -1,11 +1,6 @@
 package ca.uhn.fhir.jpa.dao;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.Validate;
@@ -52,37 +47,39 @@ public class DaoConfig {
 	// update setter javadoc if default changes
 	// ***
 	private int myDeferIndexingForCodesystemsOfSize = 2000;
+	private boolean myDeleteStaleSearches = true;
 	// ***
 	// update setter javadoc if default changes
 	// ***
 	private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR;
+	
 	private int myHardSearchLimit = 1000;
 	
 	private int myHardTagListLimit = 1000;
 	
 	private int myIncludeLimit = 2000;
-	
+
 	// ***
 	// update setter javadoc if default changes
 	// ***
 	private boolean myIndexContainedResources = true;
 
 	private List<IServerInterceptor> myInterceptors;
-
+	
 	// ***
 	// update setter javadoc if default changes
 	// ***
 	private int myMaximumExpansionSize = 5000;
-	
 	private ResourceEncodingEnum myResourceEncoding = ResourceEncodingEnum.JSONC;
+
 	private boolean mySchedulingDisabled;
 
 	private boolean mySubscriptionEnabled;
-
+	
 	private long mySubscriptionPollDelay = 1000;
 	
 	private Long mySubscriptionPurgeInactiveAfterMillis;
-	
+
 	private Set<String> myTreatBaseUrlsAsLocal = new HashSet<String>();
 
 	/**
@@ -105,6 +102,9 @@ public class DaoConfig {
 	 * number of milliseconds, they will be deleted from the database, and any paging links
 	 * (next/prev links in search response bundles) will become invalid. Defaults to 1 hour. 
 	 * </p>
+	 * <p>
+	 * @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)}
+	 * </p>
 	 * 
 	 * @since 1.5
 	 */
@@ -202,6 +202,18 @@ public class DaoConfig {
 		return myAllowMultipleDelete;
 	}
 
+	/**
+	 * If this is set to <code>false</code> (default is <code>true</code>) the stale search deletion 
+	 * task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION. 
+	 * <p>
+	 * This feature is useful if you want to define your own process for deleting these (e.g. because
+	 * you are running in a cluster)
+	 * </p>
+	 */
+	public boolean isExpireSearchResults() {
+		return myDeleteStaleSearches;
+	}
+
 	/**
 	 * Should contained IDs be indexed the same way that non-contained IDs are (default is
 	 * <code>true</code>) 
@@ -280,6 +292,18 @@ public class DaoConfig {
 		myDeferIndexingForCodesystemsOfSize = theDeferIndexingForCodesystemsOfSize;
 	}
 
+	/**
+	 * If this is set to <code>false</code> (default is <code>true</code>) the stale search deletion 
+	 * task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION. 
+	 * <p>
+	 * This feature is useful if you want to define your own process for deleting these (e.g. because
+	 * you are running in a cluster)
+	 * </p>
+	 */
+	public void setExpireSearchResults(boolean theDeleteStaleSearches) {
+		myDeleteStaleSearches = theDeleteStaleSearches;
+	}
+
 	/**
 	 * Sets the number of milliseconds that search results for a given client search 
 	 * should be preserved before being purged from the database.
@@ -289,7 +313,9 @@ public class DaoConfig {
 	 * number of milliseconds, they will be deleted from the database, and any paging links
 	 * (next/prev links in search response bundles) will become invalid. Defaults to 1 hour. 
 	 * </p>
-	 * 
+	 * <p>
+	 * @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)}
+	 * </p>
 	 * @since 1.5
 	 */
 	public void setExpireSearchResultsAfterMillis(long theExpireSearchResultsAfterMillis) {
@@ -389,11 +415,11 @@ public class DaoConfig {
 		}
 		mySubscriptionPurgeInactiveAfterMillis = theMillis;
 	}
-
+	
 	public void setSubscriptionPurgeInactiveAfterSeconds(int theSeconds) {
 		setSubscriptionPurgeInactiveAfterMillis(theSeconds * DateUtils.MILLIS_PER_SECOND);
 	}
-
+	
 	/**
 	 * This setting may be used to advise the server that any references found in
 	 * resources that have any of the base URLs given here will be replaced with

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java

@@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.search;
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  * 
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  * 
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -47,46 +47,47 @@ public class StaleSearchDeletingSvc {
 
 	@Autowired
 	private ISearchDao mySearchDao;
-	
+
 	@Autowired
 	private DaoConfig myDaoConfig;
-	
+
 	@Autowired
 	private ISearchResultDao mySearchResultDao;
-	
+
 	@Autowired
 	private ISearchIncludeDao mySearchIncludeDao;
 
 	@Autowired
 	private PlatformTransactionManager myTransactionManager;
-	
+
 	@Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND)
 	@Transactional(propagation = Propagation.NOT_SUPPORTED)
 	public synchronized void pollForStaleSearches() {
-		Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis());
+		if (myDaoConfig.isExpireSearchResults()) {
-		ourLog.debug("Searching for searches which are before {}", cutoff);
+			Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis());
-		
+			ourLog.debug("Searching for searches which are before {}", cutoff);
-		Collection<Search> toDelete = mySearchDao.findWhereCreatedBefore(cutoff);
+
-		if (toDelete.isEmpty()) {
+			Collection<Search> toDelete = mySearchDao.findWhereCreatedBefore(cutoff);
-			return;
+			if (toDelete.isEmpty()) {
+				return;
+			}
+
+			TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
+			for (final Search next : toDelete) {
+				tt.execute(new TransactionCallbackWithoutResult() {
+					@Override
+					protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
+						Search searchToDelete = mySearchDao.findOne(next.getId());
+						ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid());
+						mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
+						mySearchResultDao.deleteForSearch(searchToDelete.getId());
+						mySearchDao.delete(searchToDelete);
+					}
+				});
+			}
+
+			ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count());
 		}
-		
-		TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
-		for (final Search next : toDelete) {
-			tt.execute(new TransactionCallbackWithoutResult() {
-				@Override
-				protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
-					Search searchToDelete = mySearchDao.findOne(next.getId());
-					ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid());
-					mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
-					mySearchResultDao.deleteForSearch(searchToDelete.getId());
-					mySearchDao.delete(searchToDelete);
-				}
-			});
-		}
-		
-		ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count());
-		
 	}
-	
+
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java

@@ -3,7 +3,6 @@ package ca.uhn.fhir.jpa.dao.dstu3;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsInRelativeOrder;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasItem;
@@ -24,7 +23,6 @@ import javax.servlet.http.HttpServletRequest;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.hibernate.boot.model.source.spi.IdentifierSourceSimple;
 import org.hl7.fhir.dstu3.model.*;
 import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.dstu3.model.Bundle.BundleType;
@@ -37,23 +35,24 @@ import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.support.TransactionCallback;
 import org.springframework.transaction.support.TransactionTemplate;
 
 import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
+import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.dao.SearchParameterMap;
 import ca.uhn.fhir.jpa.dao.SearchParameterMap.EverythingModeEnum;
 import ca.uhn.fhir.jpa.entity.*;
+import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvc;
 import ca.uhn.fhir.model.api.IQueryParameterType;
 import ca.uhn.fhir.model.api.Include;
-import ca.uhn.fhir.model.dstu.composite.ResourceReferenceDt;
-import ca.uhn.fhir.model.dstu.resource.MedicationPrescription;
 import ca.uhn.fhir.model.dstu.valueset.QuantityCompararatorEnum;
-import ca.uhn.fhir.parser.DataFormatException;
 import ca.uhn.fhir.rest.api.SortOrderEnum;
 import ca.uhn.fhir.rest.api.SortSpec;
 import ca.uhn.fhir.rest.param.*;
@@ -1504,6 +1503,80 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
 
 	}
 
+	@After
+	public final void after() {
+		myDaoConfig.setExpireSearchResults(new DaoConfig().isExpireSearchResults());
+		myDaoConfig.setExpireSearchResultsAfterMillis(new DaoConfig().getExpireSearchResultsAfterMillis());
+	}
+	
+	@Autowired
+	protected StaleSearchDeletingSvc myStaleSearchDeletingSvc;
+	
+
+	@Test
+	public void testSearchPagesExpiryDisabled() throws Exception {
+		IIdType pid1;
+		IIdType pid2;
+		{
+			Patient patient = new Patient();
+			patient.addName().addFamily("EXPIRE");
+			pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+		}
+		Date between = new Date();
+		Thread.sleep(10);
+		{
+			Patient patient = new Patient();
+			patient.addName().addFamily("EXPIRE");
+			pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+		}
+		Thread.sleep(10);
+		
+		SearchParameterMap params;
+		params = new SearchParameterMap();
+		params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
+		IBundleProvider bundleProvider = myPatientDao.search(params);
+		assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+		assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+
+		myDaoConfig.setExpireSearchResults(false);
+		myStaleSearchDeletingSvc.pollForStaleSearches();
+		Thread.sleep(1500);
+		
+		assertThat(toUnqualifiedVersionlessIds(bundleProvider), (containsInAnyOrder(pid1, pid2)));
+	}
+
+	@Test
+	public void testSearchPagesExpiry() throws Exception {
+		IIdType pid1;
+		IIdType pid2;
+		{
+			Patient patient = new Patient();
+			patient.addName().addFamily("EXPIRE");
+			pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+		}
+		Thread.sleep(10);
+		{
+			Patient patient = new Patient();
+			patient.addName().addFamily("EXPIRE");
+			pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+		}
+		Thread.sleep(10);
+
+		SearchParameterMap params;
+		params = new SearchParameterMap();
+		params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
+		IBundleProvider bundleProvider = myPatientDao.search(params);
+		assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+		assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+
+		Thread.sleep(1500);
+		
+		myDaoConfig.setExpireSearchResultsAfterMillis(500);
+		myStaleSearchDeletingSvc.pollForStaleSearches();
+		
+		assertThat(toUnqualifiedVersionlessIds(bundleProvider), not(containsInAnyOrder(pid1, pid2)));
+	}
+
 	@Test
 	public void testSearchStringParamReallyLong() {
 		String methodName = "testSearchStringParamReallyLong";

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java

@@ -3,13 +3,18 @@ package ca.uhn.fhir.jpa.provider.dstu3;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import javax.servlet.DispatcherType;
+
+import org.apache.catalina.filters.CorsFilter;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.hl7.fhir.dstu3.hapi.validation.DefaultProfileValidationSupport;
@@ -36,6 +41,7 @@ import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
 import ca.uhn.fhir.rest.client.IGenericClient;
 import ca.uhn.fhir.rest.client.ServerValidationModeEnum;
 import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
+import ca.uhn.fhir.rest.server.CORSFilter_;
 import ca.uhn.fhir.rest.server.FifoMemoryPagingProvider;
 import ca.uhn.fhir.rest.server.RestfulServer;
 import ca.uhn.fhir.util.TestUtil;
@@ -114,6 +120,15 @@ public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test {
 			subsServletHolder.setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, WebsocketDstu3Config.class.getName());
 			proxyHandler.addServlet(subsServletHolder, "/*");
 
+			FilterHolder corsFilterHolder = new FilterHolder();
+			corsFilterHolder.setHeldClass(CorsFilter.class);
+			corsFilterHolder.setInitParameter("cors.allowed.origins", "*");
+			corsFilterHolder.setInitParameter("cors.allowed.methods", "GET,POST,PUT,DELETE,OPTIONS");
+			corsFilterHolder.setInitParameter("cors.allowed.headers", "X-FHIR-Starter,Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers");
+			corsFilterHolder.setInitParameter("cors.exposed.headers", "Location,Content-Location");
+			corsFilterHolder.setInitParameter("cors.support.credentials", "true");
+			corsFilterHolder.setInitParameter("cors.logging.enabled", "true");
+			proxyHandler.addFilter(corsFilterHolder, "/*", EnumSet.of(DispatcherType.REQUEST));
 			
 			server.setHandler(proxyHandler);
 			server.start();

src/changes/changes.xml

@@ -19,6 +19,12 @@
 				(e.g. "StructureDefinition.url"). Thanks
 				to David Hay for reporting!
 			</action>
+			<action type="add">
+				Add ability to JPA server for disabling stale search
+				expiry. This is useful if you are deploying the server
+				to a cluster.
+			</action>
+		</release>
 		<release version="2.1" date="2016-11-11">
 			<action type="add">
 				STU3 structure definitions have been updated to the