From ca9223fb70cbd450db0536218480c2a4cecd292f Mon Sep 17 00:00:00 2001 From: James Agnew Date: Thu, 17 Nov 2016 10:01:23 +0100 Subject: [PATCH] Add ability to disable stale search expiry --- .../java/ca/uhn/fhir/jpa/dao/DaoConfig.java | 54 ++++++++---- .../jpa/search/StaleSearchDeletingSvc.java | 59 ++++++------- .../FhirResourceDaoDstu3SearchNoFtTest.java | 83 +++++++++++++++++-- .../dstu3/BaseResourceProviderDstu3Test.java | 15 ++++ src/changes/changes.xml | 6 ++ 5 files changed, 169 insertions(+), 48 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java index b92f62b22cc..b32a1f3fbdd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java @@ -1,11 +1,6 @@ package ca.uhn.fhir.jpa.dao; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.Validate; @@ -52,37 +47,39 @@ public class DaoConfig { // update setter javadoc if default changes // *** private int myDeferIndexingForCodesystemsOfSize = 2000; + private boolean myDeleteStaleSearches = true; // *** // update setter javadoc if default changes // *** private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR; + private int myHardSearchLimit = 1000; private int myHardTagListLimit = 1000; private int myIncludeLimit = 2000; - + // *** // update setter javadoc if default changes // *** private boolean myIndexContainedResources = true; private List myInterceptors; - + // *** // update setter javadoc if default changes // *** private int myMaximumExpansionSize = 5000; - private ResourceEncodingEnum myResourceEncoding = ResourceEncodingEnum.JSONC; + private boolean mySchedulingDisabled; private boolean mySubscriptionEnabled; - + private long mySubscriptionPollDelay = 1000; private Long mySubscriptionPurgeInactiveAfterMillis; - + private Set myTreatBaseUrlsAsLocal = new HashSet(); /** @@ -105,6 +102,9 @@ public class DaoConfig { * number of milliseconds, they will be deleted from the database, and any paging links * (next/prev links in search response bundles) will become invalid. Defaults to 1 hour. *

+ *

+ * @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)} + *

* * @since 1.5 */ @@ -202,6 +202,18 @@ public class DaoConfig { return myAllowMultipleDelete; } + /** + * If this is set to false (default is true) the stale search deletion + * task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION. + *

+ * This feature is useful if you want to define your own process for deleting these (e.g. because + * you are running in a cluster) + *

+ */ + public boolean isExpireSearchResults() { + return myDeleteStaleSearches; + } + /** * Should contained IDs be indexed the same way that non-contained IDs are (default is * true) @@ -280,6 +292,18 @@ public class DaoConfig { myDeferIndexingForCodesystemsOfSize = theDeferIndexingForCodesystemsOfSize; } + /** + * If this is set to false (default is true) the stale search deletion + * task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION. + *

+ * This feature is useful if you want to define your own process for deleting these (e.g. because + * you are running in a cluster) + *

+ */ + public void setExpireSearchResults(boolean theDeleteStaleSearches) { + myDeleteStaleSearches = theDeleteStaleSearches; + } + /** * Sets the number of milliseconds that search results for a given client search * should be preserved before being purged from the database. @@ -289,7 +313,9 @@ public class DaoConfig { * number of milliseconds, they will be deleted from the database, and any paging links * (next/prev links in search response bundles) will become invalid. Defaults to 1 hour. *

- * + *

+ * @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)} + *

* @since 1.5 */ public void setExpireSearchResultsAfterMillis(long theExpireSearchResultsAfterMillis) { @@ -389,11 +415,11 @@ public class DaoConfig { } mySubscriptionPurgeInactiveAfterMillis = theMillis; } - + public void setSubscriptionPurgeInactiveAfterSeconds(int theSeconds) { setSubscriptionPurgeInactiveAfterMillis(theSeconds * DateUtils.MILLIS_PER_SECOND); } - + /** * This setting may be used to advise the server that any references found in * resources that have any of the base URLs given here will be replaced with diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java index d3bc9842339..e7a38b32534 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvc.java @@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.search; * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -47,46 +47,47 @@ public class StaleSearchDeletingSvc { @Autowired private ISearchDao mySearchDao; - + @Autowired private DaoConfig myDaoConfig; - + @Autowired private ISearchResultDao mySearchResultDao; - + @Autowired private ISearchIncludeDao mySearchIncludeDao; @Autowired private PlatformTransactionManager myTransactionManager; - + @Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND) @Transactional(propagation = Propagation.NOT_SUPPORTED) public synchronized void pollForStaleSearches() { - Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis()); - ourLog.debug("Searching for searches which are before {}", cutoff); - - Collection toDelete = mySearchDao.findWhereCreatedBefore(cutoff); - if (toDelete.isEmpty()) { - return; + if (myDaoConfig.isExpireSearchResults()) { + Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis()); + ourLog.debug("Searching for searches which are before {}", cutoff); + + Collection toDelete = mySearchDao.findWhereCreatedBefore(cutoff); + if (toDelete.isEmpty()) { + return; + } + + TransactionTemplate tt = new TransactionTemplate(myTransactionManager); + for (final Search next : toDelete) { + tt.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(TransactionStatus theArg0) { + Search searchToDelete = mySearchDao.findOne(next.getId()); + ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid()); + mySearchIncludeDao.deleteForSearch(searchToDelete.getId()); + mySearchResultDao.deleteForSearch(searchToDelete.getId()); + mySearchDao.delete(searchToDelete); + } + }); + } + + ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count()); } - - TransactionTemplate tt = new TransactionTemplate(myTransactionManager); - for (final Search next : toDelete) { - tt.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus theArg0) { - Search searchToDelete = mySearchDao.findOne(next.getId()); - ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid()); - mySearchIncludeDao.deleteForSearch(searchToDelete.getId()); - mySearchResultDao.deleteForSearch(searchToDelete.getId()); - mySearchDao.delete(searchToDelete); - } - }); - } - - ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count()); - } - + } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java index 04db9cb6471..3bd36c5975b 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3SearchNoFtTest.java @@ -3,7 +3,6 @@ package ca.uhn.fhir.jpa.dao.dstu3; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsInRelativeOrder; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.hasItem; @@ -24,7 +23,6 @@ import javax.servlet.http.HttpServletRequest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.hibernate.boot.model.source.spi.IdentifierSourceSimple; import org.hl7.fhir.dstu3.model.*; import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent; import org.hl7.fhir.dstu3.model.Bundle.BundleType; @@ -37,23 +35,24 @@ import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.junit.After; import org.junit.AfterClass; import org.junit.Ignore; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; +import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.SearchParameterMap; import ca.uhn.fhir.jpa.dao.SearchParameterMap.EverythingModeEnum; import ca.uhn.fhir.jpa.entity.*; +import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvc; import ca.uhn.fhir.model.api.IQueryParameterType; import ca.uhn.fhir.model.api.Include; -import ca.uhn.fhir.model.dstu.composite.ResourceReferenceDt; -import ca.uhn.fhir.model.dstu.resource.MedicationPrescription; import ca.uhn.fhir.model.dstu.valueset.QuantityCompararatorEnum; -import ca.uhn.fhir.parser.DataFormatException; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.param.*; @@ -1504,6 +1503,80 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test { } + @After + public final void after() { + myDaoConfig.setExpireSearchResults(new DaoConfig().isExpireSearchResults()); + myDaoConfig.setExpireSearchResultsAfterMillis(new DaoConfig().getExpireSearchResultsAfterMillis()); + } + + @Autowired + protected StaleSearchDeletingSvc myStaleSearchDeletingSvc; + + + @Test + public void testSearchPagesExpiryDisabled() throws Exception { + IIdType pid1; + IIdType pid2; + { + Patient patient = new Patient(); + patient.addName().addFamily("EXPIRE"); + pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + } + Date between = new Date(); + Thread.sleep(10); + { + Patient patient = new Patient(); + patient.addName().addFamily("EXPIRE"); + pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + } + Thread.sleep(10); + + SearchParameterMap params; + params = new SearchParameterMap(); + params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); + IBundleProvider bundleProvider = myPatientDao.search(params); + assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); + assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); + + myDaoConfig.setExpireSearchResults(false); + myStaleSearchDeletingSvc.pollForStaleSearches(); + Thread.sleep(1500); + + assertThat(toUnqualifiedVersionlessIds(bundleProvider), (containsInAnyOrder(pid1, pid2))); + } + + @Test + public void testSearchPagesExpiry() throws Exception { + IIdType pid1; + IIdType pid2; + { + Patient patient = new Patient(); + patient.addName().addFamily("EXPIRE"); + pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + } + Thread.sleep(10); + { + Patient patient = new Patient(); + patient.addName().addFamily("EXPIRE"); + pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + } + Thread.sleep(10); + + SearchParameterMap params; + params = new SearchParameterMap(); + params.add(Patient.SP_FAMILY, new StringParam("EXPIRE")); + IBundleProvider bundleProvider = myPatientDao.search(params); + assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); + assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2)); + + Thread.sleep(1500); + + myDaoConfig.setExpireSearchResultsAfterMillis(500); + myStaleSearchDeletingSvc.pollForStaleSearches(); + + assertThat(toUnqualifiedVersionlessIds(bundleProvider), not(containsInAnyOrder(pid1, pid2))); + } + @Test public void testSearchStringParamReallyLong() { String methodName = "testSearchStringParamReallyLong"; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java index 71723bdfc0c..64a22692e39 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/BaseResourceProviderDstu3Test.java @@ -3,13 +3,18 @@ package ca.uhn.fhir.jpa.provider.dstu3; import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.concurrent.TimeUnit; +import javax.servlet.DispatcherType; + +import org.apache.catalina.filters.CorsFilter; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.hl7.fhir.dstu3.hapi.validation.DefaultProfileValidationSupport; @@ -36,6 +41,7 @@ import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator; import ca.uhn.fhir.rest.client.IGenericClient; import ca.uhn.fhir.rest.client.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; +import ca.uhn.fhir.rest.server.CORSFilter_; import ca.uhn.fhir.rest.server.FifoMemoryPagingProvider; import ca.uhn.fhir.rest.server.RestfulServer; import ca.uhn.fhir.util.TestUtil; @@ -114,6 +120,15 @@ public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test { subsServletHolder.setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, WebsocketDstu3Config.class.getName()); proxyHandler.addServlet(subsServletHolder, "/*"); + FilterHolder corsFilterHolder = new FilterHolder(); + corsFilterHolder.setHeldClass(CorsFilter.class); + corsFilterHolder.setInitParameter("cors.allowed.origins", "*"); + corsFilterHolder.setInitParameter("cors.allowed.methods", "GET,POST,PUT,DELETE,OPTIONS"); + corsFilterHolder.setInitParameter("cors.allowed.headers", "X-FHIR-Starter,Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers"); + corsFilterHolder.setInitParameter("cors.exposed.headers", "Location,Content-Location"); + corsFilterHolder.setInitParameter("cors.support.credentials", "true"); + corsFilterHolder.setInitParameter("cors.logging.enabled", "true"); + proxyHandler.addFilter(corsFilterHolder, "/*", EnumSet.of(DispatcherType.REQUEST)); server.setHandler(proxyHandler); server.start(); diff --git a/src/changes/changes.xml b/src/changes/changes.xml index d4622728249..b87602a42d2 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -19,6 +19,12 @@ (e.g. "StructureDefinition.url"). Thanks to David Hay for reporting! + + Add ability to JPA server for disabling stale search + expiry. This is useful if you are deploying the server + to a cluster. + + STU3 structure definitions have been updated to the