From ca9223fb70cbd450db0536218480c2a4cecd292f Mon Sep 17 00:00:00 2001
From: James Agnew
Date: Thu, 17 Nov 2016 10:01:23 +0100
Subject: [PATCH] Add ability to disable stale search expiry
---
.../java/ca/uhn/fhir/jpa/dao/DaoConfig.java | 54 ++++++++----
.../jpa/search/StaleSearchDeletingSvc.java | 59 ++++++-------
.../FhirResourceDaoDstu3SearchNoFtTest.java | 83 +++++++++++++++++--
.../dstu3/BaseResourceProviderDstu3Test.java | 15 ++++
src/changes/changes.xml | 6 ++
5 files changed, 169 insertions(+), 48 deletions(-)
diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
index b92f62b22cc..b32a1f3fbdd 100644
--- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
+++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
@@ -1,11 +1,6 @@
package ca.uhn.fhir.jpa.dao;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.Validate;
@@ -52,37 +47,39 @@ public class DaoConfig {
// update setter javadoc if default changes
// ***
private int myDeferIndexingForCodesystemsOfSize = 2000;
+ private boolean myDeleteStaleSearches = true;
// ***
// update setter javadoc if default changes
// ***
private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR;
+
private int myHardSearchLimit = 1000;
private int myHardTagListLimit = 1000;
private int myIncludeLimit = 2000;
-
+
// ***
// update setter javadoc if default changes
// ***
private boolean myIndexContainedResources = true;
private List myInterceptors;
-
+
// ***
// update setter javadoc if default changes
// ***
private int myMaximumExpansionSize = 5000;
-
private ResourceEncodingEnum myResourceEncoding = ResourceEncodingEnum.JSONC;
+
private boolean mySchedulingDisabled;
private boolean mySubscriptionEnabled;
-
+
private long mySubscriptionPollDelay = 1000;
private Long mySubscriptionPurgeInactiveAfterMillis;
-
+
private Set myTreatBaseUrlsAsLocal = new HashSet();
/**
@@ -105,6 +102,9 @@ public class DaoConfig {
* number of milliseconds, they will be deleted from the database, and any paging links
* (next/prev links in search response bundles) will become invalid. Defaults to 1 hour.
*
+ *
+ * @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)}
+ *
*
* @since 1.5
*/
@@ -202,6 +202,18 @@ public class DaoConfig {
return myAllowMultipleDelete;
}
+ /**
+ * If this is set to false
(default is true
) the stale search deletion
+ * task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION.
+ *
+ * This feature is useful if you want to define your own process for deleting these (e.g. because
+ * you are running in a cluster)
+ *
+ */
+ public boolean isExpireSearchResults() {
+ return myDeleteStaleSearches;
+ }
+
/**
* Should contained IDs be indexed the same way that non-contained IDs are (default is
* true
)
@@ -280,6 +292,18 @@ public class DaoConfig {
myDeferIndexingForCodesystemsOfSize = theDeferIndexingForCodesystemsOfSize;
}
+ /**
+ * If this is set to false
(default is true
) the stale search deletion
+ * task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION.
+ *
+ * This feature is useful if you want to define your own process for deleting these (e.g. because
+ * you are running in a cluster)
+ *
+ */
+ public void setExpireSearchResults(boolean theDeleteStaleSearches) {
+ myDeleteStaleSearches = theDeleteStaleSearches;
+ }
+
/**
* Sets the number of milliseconds that search results for a given client search
* should be preserved before being purged from the database.
@@ -289,7 +313,9 @@ public class DaoConfig {
* number of milliseconds, they will be deleted from the database, and any paging links
* (next/prev links in search response bundles) will become invalid. Defaults to 1 hour.
*
- *
+ *
+ * @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)}
+ *
* @since 1.5
*/
public void setExpireSearchResultsAfterMillis(long theExpireSearchResultsAfterMillis) {
@@ -389,11 +415,11 @@ public class DaoConfig {
}
mySubscriptionPurgeInactiveAfterMillis = theMillis;
}
-
+
public void setSubscriptionPurgeInactiveAfterSeconds(int theSeconds) {
setSubscriptionPurgeInactiveAfterMillis(theSeconds * DateUtils.MILLIS_PER_SECOND);
}
-
+
/**
* This setting may be used to advise the server that any references found in
* resources that have any of the base URLs given here will be replaced with
diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java
index d3bc9842339..e7a38b32534 100644
--- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java
+++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java
@@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.search;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -47,46 +47,47 @@ public class StaleSearchDeletingSvc {
@Autowired
private ISearchDao mySearchDao;
-
+
@Autowired
private DaoConfig myDaoConfig;
-
+
@Autowired
private ISearchResultDao mySearchResultDao;
-
+
@Autowired
private ISearchIncludeDao mySearchIncludeDao;
@Autowired
private PlatformTransactionManager myTransactionManager;
-
+
@Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public synchronized void pollForStaleSearches() {
- Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis());
- ourLog.debug("Searching for searches which are before {}", cutoff);
-
- Collection toDelete = mySearchDao.findWhereCreatedBefore(cutoff);
- if (toDelete.isEmpty()) {
- return;
+ if (myDaoConfig.isExpireSearchResults()) {
+ Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis());
+ ourLog.debug("Searching for searches which are before {}", cutoff);
+
+ Collection toDelete = mySearchDao.findWhereCreatedBefore(cutoff);
+ if (toDelete.isEmpty()) {
+ return;
+ }
+
+ TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
+ for (final Search next : toDelete) {
+ tt.execute(new TransactionCallbackWithoutResult() {
+ @Override
+ protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
+ Search searchToDelete = mySearchDao.findOne(next.getId());
+ ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid());
+ mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
+ mySearchResultDao.deleteForSearch(searchToDelete.getId());
+ mySearchDao.delete(searchToDelete);
+ }
+ });
+ }
+
+ ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count());
}
-
- TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
- for (final Search next : toDelete) {
- tt.execute(new TransactionCallbackWithoutResult() {
- @Override
- protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
- Search searchToDelete = mySearchDao.findOne(next.getId());
- ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid());
- mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
- mySearchResultDao.deleteForSearch(searchToDelete.getId());
- mySearchDao.delete(searchToDelete);
- }
- });
- }
-
- ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count());
-
}
-
+
}
diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java
index 04db9cb6471..3bd36c5975b 100644
--- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java
+++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java
@@ -3,7 +3,6 @@ package ca.uhn.fhir.jpa.dao.dstu3;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsInRelativeOrder;
-import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.hasItem;
@@ -24,7 +23,6 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
-import org.hibernate.boot.model.source.spi.IdentifierSourceSimple;
import org.hl7.fhir.dstu3.model.*;
import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.dstu3.model.Bundle.BundleType;
@@ -37,23 +35,24 @@ import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.instance.model.api.IPrimitiveType;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Ignore;
import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
+import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.dao.SearchParameterMap;
import ca.uhn.fhir.jpa.dao.SearchParameterMap.EverythingModeEnum;
import ca.uhn.fhir.jpa.entity.*;
+import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvc;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.Include;
-import ca.uhn.fhir.model.dstu.composite.ResourceReferenceDt;
-import ca.uhn.fhir.model.dstu.resource.MedicationPrescription;
import ca.uhn.fhir.model.dstu.valueset.QuantityCompararatorEnum;
-import ca.uhn.fhir.parser.DataFormatException;
import ca.uhn.fhir.rest.api.SortOrderEnum;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.param.*;
@@ -1504,6 +1503,80 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
}
+ @After
+ public final void after() {
+ myDaoConfig.setExpireSearchResults(new DaoConfig().isExpireSearchResults());
+ myDaoConfig.setExpireSearchResultsAfterMillis(new DaoConfig().getExpireSearchResultsAfterMillis());
+ }
+
+ @Autowired
+ protected StaleSearchDeletingSvc myStaleSearchDeletingSvc;
+
+
+ @Test
+ public void testSearchPagesExpiryDisabled() throws Exception {
+ IIdType pid1;
+ IIdType pid2;
+ {
+ Patient patient = new Patient();
+ patient.addName().addFamily("EXPIRE");
+ pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+ }
+ Date between = new Date();
+ Thread.sleep(10);
+ {
+ Patient patient = new Patient();
+ patient.addName().addFamily("EXPIRE");
+ pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+ }
+ Thread.sleep(10);
+
+ SearchParameterMap params;
+ params = new SearchParameterMap();
+ params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
+ IBundleProvider bundleProvider = myPatientDao.search(params);
+ assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+ assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+
+ myDaoConfig.setExpireSearchResults(false);
+ myStaleSearchDeletingSvc.pollForStaleSearches();
+ Thread.sleep(1500);
+
+ assertThat(toUnqualifiedVersionlessIds(bundleProvider), (containsInAnyOrder(pid1, pid2)));
+ }
+
+ @Test
+ public void testSearchPagesExpiry() throws Exception {
+ IIdType pid1;
+ IIdType pid2;
+ {
+ Patient patient = new Patient();
+ patient.addName().addFamily("EXPIRE");
+ pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+ }
+ Thread.sleep(10);
+ {
+ Patient patient = new Patient();
+ patient.addName().addFamily("EXPIRE");
+ pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
+ }
+ Thread.sleep(10);
+
+ SearchParameterMap params;
+ params = new SearchParameterMap();
+ params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
+ IBundleProvider bundleProvider = myPatientDao.search(params);
+ assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+ assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
+
+ Thread.sleep(1500);
+
+ myDaoConfig.setExpireSearchResultsAfterMillis(500);
+ myStaleSearchDeletingSvc.pollForStaleSearches();
+
+ assertThat(toUnqualifiedVersionlessIds(bundleProvider), not(containsInAnyOrder(pid1, pid2)));
+ }
+
@Test
public void testSearchStringParamReallyLong() {
String methodName = "testSearchStringParamReallyLong";
diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java
index 71723bdfc0c..64a22692e39 100644
--- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java
+++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java
@@ -3,13 +3,18 @@ package ca.uhn.fhir.jpa.provider.dstu3;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import javax.servlet.DispatcherType;
+
+import org.apache.catalina.filters.CorsFilter;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.hl7.fhir.dstu3.hapi.validation.DefaultProfileValidationSupport;
@@ -36,6 +41,7 @@ import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
import ca.uhn.fhir.rest.client.IGenericClient;
import ca.uhn.fhir.rest.client.ServerValidationModeEnum;
import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
+import ca.uhn.fhir.rest.server.CORSFilter_;
import ca.uhn.fhir.rest.server.FifoMemoryPagingProvider;
import ca.uhn.fhir.rest.server.RestfulServer;
import ca.uhn.fhir.util.TestUtil;
@@ -114,6 +120,15 @@ public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test {
subsServletHolder.setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, WebsocketDstu3Config.class.getName());
proxyHandler.addServlet(subsServletHolder, "/*");
+ FilterHolder corsFilterHolder = new FilterHolder();
+ corsFilterHolder.setHeldClass(CorsFilter.class);
+ corsFilterHolder.setInitParameter("cors.allowed.origins", "*");
+ corsFilterHolder.setInitParameter("cors.allowed.methods", "GET,POST,PUT,DELETE,OPTIONS");
+ corsFilterHolder.setInitParameter("cors.allowed.headers", "X-FHIR-Starter,Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers");
+ corsFilterHolder.setInitParameter("cors.exposed.headers", "Location,Content-Location");
+ corsFilterHolder.setInitParameter("cors.support.credentials", "true");
+ corsFilterHolder.setInitParameter("cors.logging.enabled", "true");
+ proxyHandler.addFilter(corsFilterHolder, "/*", EnumSet.of(DispatcherType.REQUEST));
server.setHandler(proxyHandler);
server.start();
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index d4622728249..b87602a42d2 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -19,6 +19,12 @@
(e.g. "StructureDefinition.url"). Thanks
to David Hay for reporting!
+
+ Add ability to JPA server for disabling stale search
+ expiry. This is useful if you are deploying the server
+ to a cluster.
+
+
STU3 structure definitions have been updated to the