Add ability to disable stale search expiry
This commit is contained in:
James Agnew
parent
391999062e
commit
ca9223fb70
|
|
@ -1,11 +1,6 @@
|
|||
package ca.uhn.fhir.jpa.dao;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.commons.lang3.ObjectUtils;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
|
@ -52,37 +47,39 @@ public class DaoConfig {
|
|||
// update setter javadoc if default changes
|
||||
// ***
|
||||
private int myDeferIndexingForCodesystemsOfSize = 2000;
|
||||
private boolean myDeleteStaleSearches = true;
|
||||
// ***
|
||||
// update setter javadoc if default changes
|
||||
// ***
|
||||
private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR;
|
||||
|
||||
private int myHardSearchLimit = 1000;
|
||||
|
||||
private int myHardTagListLimit = 1000;
|
||||
|
||||
private int myIncludeLimit = 2000;
|
||||
|
||||
|
||||
// ***
|
||||
// update setter javadoc if default changes
|
||||
// ***
|
||||
private boolean myIndexContainedResources = true;
|
||||
|
||||
private List<IServerInterceptor> myInterceptors;
|
||||
|
||||
|
||||
// ***
|
||||
// update setter javadoc if default changes
|
||||
// ***
|
||||
private int myMaximumExpansionSize = 5000;
|
||||
|
||||
private ResourceEncodingEnum myResourceEncoding = ResourceEncodingEnum.JSONC;
|
||||
|
||||
private boolean mySchedulingDisabled;
|
||||
|
||||
private boolean mySubscriptionEnabled;
|
||||
|
||||
|
||||
private long mySubscriptionPollDelay = 1000;
|
||||
|
||||
private Long mySubscriptionPurgeInactiveAfterMillis;
|
||||
|
||||
|
||||
private Set<String> myTreatBaseUrlsAsLocal = new HashSet<String>();
|
||||
|
||||
/**
|
||||
|
|
@ -105,6 +102,9 @@ public class DaoConfig {
|
|||
* number of milliseconds, they will be deleted from the database, and any paging links
|
||||
* (next/prev links in search response bundles) will become invalid. Defaults to 1 hour.
|
||||
* </p>
|
||||
* <p>
|
||||
* @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)}
|
||||
* </p>
|
||||
*
|
||||
* @since 1.5
|
||||
*/
|
||||
|
|
@ -202,6 +202,18 @@ public class DaoConfig {
|
|||
return myAllowMultipleDelete;
|
||||
}
|
||||
|
||||
/**
|
||||
* If this is set to <code>false</code> (default is <code>true</code>) the stale search deletion
|
||||
* task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION.
|
||||
* <p>
|
||||
* This feature is useful if you want to define your own process for deleting these (e.g. because
|
||||
* you are running in a cluster)
|
||||
* </p>
|
||||
*/
|
||||
public boolean isExpireSearchResults() {
|
||||
return myDeleteStaleSearches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Should contained IDs be indexed the same way that non-contained IDs are (default is
|
||||
* <code>true</code>)
|
||||
|
|
@ -280,6 +292,18 @@ public class DaoConfig {
|
|||
myDeferIndexingForCodesystemsOfSize = theDeferIndexingForCodesystemsOfSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* If this is set to <code>false</code> (default is <code>true</code>) the stale search deletion
|
||||
* task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION.
|
||||
* <p>
|
||||
* This feature is useful if you want to define your own process for deleting these (e.g. because
|
||||
* you are running in a cluster)
|
||||
* </p>
|
||||
*/
|
||||
public void setExpireSearchResults(boolean theDeleteStaleSearches) {
|
||||
myDeleteStaleSearches = theDeleteStaleSearches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the number of milliseconds that search results for a given client search
|
||||
* should be preserved before being purged from the database.
|
||||
|
|
@ -289,7 +313,9 @@ public class DaoConfig {
|
|||
* number of milliseconds, they will be deleted from the database, and any paging links
|
||||
* (next/prev links in search response bundles) will become invalid. Defaults to 1 hour.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* @see To disable this feature entirely, see {@link #setExpireSearchResults(boolean)}
|
||||
* </p>
|
||||
* @since 1.5
|
||||
*/
|
||||
public void setExpireSearchResultsAfterMillis(long theExpireSearchResultsAfterMillis) {
|
||||
|
|
@ -389,11 +415,11 @@ public class DaoConfig {
|
|||
}
|
||||
mySubscriptionPurgeInactiveAfterMillis = theMillis;
|
||||
}
|
||||
|
||||
|
||||
public void setSubscriptionPurgeInactiveAfterSeconds(int theSeconds) {
|
||||
setSubscriptionPurgeInactiveAfterMillis(theSeconds * DateUtils.MILLIS_PER_SECOND);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This setting may be used to advise the server that any references found in
|
||||
* resources that have any of the base URLs given here will be replaced with
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ package ca.uhn.fhir.jpa.search;
|
|||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
|
@ -47,46 +47,47 @@ public class StaleSearchDeletingSvc {
|
|||
|
||||
@Autowired
|
||||
private ISearchDao mySearchDao;
|
||||
|
||||
|
||||
@Autowired
|
||||
private DaoConfig myDaoConfig;
|
||||
|
||||
|
||||
@Autowired
|
||||
private ISearchResultDao mySearchResultDao;
|
||||
|
||||
|
||||
@Autowired
|
||||
private ISearchIncludeDao mySearchIncludeDao;
|
||||
|
||||
@Autowired
|
||||
private PlatformTransactionManager myTransactionManager;
|
||||
|
||||
|
||||
@Scheduled(fixedDelay = 10 * DateUtils.MILLIS_PER_SECOND)
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
public synchronized void pollForStaleSearches() {
|
||||
Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis());
|
||||
ourLog.debug("Searching for searches which are before {}", cutoff);
|
||||
|
||||
Collection<Search> toDelete = mySearchDao.findWhereCreatedBefore(cutoff);
|
||||
if (toDelete.isEmpty()) {
|
||||
return;
|
||||
if (myDaoConfig.isExpireSearchResults()) {
|
||||
Date cutoff = new Date(System.currentTimeMillis() - myDaoConfig.getExpireSearchResultsAfterMillis());
|
||||
ourLog.debug("Searching for searches which are before {}", cutoff);
|
||||
|
||||
Collection<Search> toDelete = mySearchDao.findWhereCreatedBefore(cutoff);
|
||||
if (toDelete.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
|
||||
for (final Search next : toDelete) {
|
||||
tt.execute(new TransactionCallbackWithoutResult() {
|
||||
@Override
|
||||
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
|
||||
Search searchToDelete = mySearchDao.findOne(next.getId());
|
||||
ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid());
|
||||
mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
|
||||
mySearchResultDao.deleteForSearch(searchToDelete.getId());
|
||||
mySearchDao.delete(searchToDelete);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count());
|
||||
}
|
||||
|
||||
TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
|
||||
for (final Search next : toDelete) {
|
||||
tt.execute(new TransactionCallbackWithoutResult() {
|
||||
@Override
|
||||
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
|
||||
Search searchToDelete = mySearchDao.findOne(next.getId());
|
||||
ourLog.info("Expiring stale search {} / {}", searchToDelete.getId(), searchToDelete.getUuid());
|
||||
mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
|
||||
mySearchResultDao.deleteForSearch(searchToDelete.getId());
|
||||
mySearchDao.delete(searchToDelete);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
ourLog.info("Deleted {} searches, {} remaining", toDelete.size(), mySearchDao.count());
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package ca.uhn.fhir.jpa.dao.dstu3;
|
|||
import static org.hamcrest.Matchers.contains;
|
||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||
import static org.hamcrest.Matchers.containsInRelativeOrder;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.empty;
|
||||
import static org.hamcrest.Matchers.endsWith;
|
||||
import static org.hamcrest.Matchers.hasItem;
|
||||
|
|
@ -24,7 +23,6 @@ import javax.servlet.http.HttpServletRequest;
|
|||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.hibernate.boot.model.source.spi.IdentifierSourceSimple;
|
||||
import org.hl7.fhir.dstu3.model.*;
|
||||
import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
|
||||
import org.hl7.fhir.dstu3.model.Bundle.BundleType;
|
||||
|
|
@ -37,23 +35,24 @@ import org.hl7.fhir.instance.model.api.IAnyResource;
|
|||
import org.hl7.fhir.instance.model.api.IBaseResource;
|
||||
import org.hl7.fhir.instance.model.api.IIdType;
|
||||
import org.hl7.fhir.instance.model.api.IPrimitiveType;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import org.springframework.transaction.support.TransactionCallback;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
|
||||
import ca.uhn.fhir.jpa.dao.DaoConfig;
|
||||
import ca.uhn.fhir.jpa.dao.SearchParameterMap;
|
||||
import ca.uhn.fhir.jpa.dao.SearchParameterMap.EverythingModeEnum;
|
||||
import ca.uhn.fhir.jpa.entity.*;
|
||||
import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvc;
|
||||
import ca.uhn.fhir.model.api.IQueryParameterType;
|
||||
import ca.uhn.fhir.model.api.Include;
|
||||
import ca.uhn.fhir.model.dstu.composite.ResourceReferenceDt;
|
||||
import ca.uhn.fhir.model.dstu.resource.MedicationPrescription;
|
||||
import ca.uhn.fhir.model.dstu.valueset.QuantityCompararatorEnum;
|
||||
import ca.uhn.fhir.parser.DataFormatException;
|
||||
import ca.uhn.fhir.rest.api.SortOrderEnum;
|
||||
import ca.uhn.fhir.rest.api.SortSpec;
|
||||
import ca.uhn.fhir.rest.param.*;
|
||||
|
|
@ -1504,6 +1503,80 @@ public class FhirResourceDaoDstu3SearchNoFtTest extends BaseJpaDstu3Test {
|
|||
|
||||
}
|
||||
|
||||
@After
|
||||
public final void after() {
|
||||
myDaoConfig.setExpireSearchResults(new DaoConfig().isExpireSearchResults());
|
||||
myDaoConfig.setExpireSearchResultsAfterMillis(new DaoConfig().getExpireSearchResultsAfterMillis());
|
||||
}
|
||||
|
||||
@Autowired
|
||||
protected StaleSearchDeletingSvc myStaleSearchDeletingSvc;
|
||||
|
||||
|
||||
@Test
|
||||
public void testSearchPagesExpiryDisabled() throws Exception {
|
||||
IIdType pid1;
|
||||
IIdType pid2;
|
||||
{
|
||||
Patient patient = new Patient();
|
||||
patient.addName().addFamily("EXPIRE");
|
||||
pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
|
||||
}
|
||||
Date between = new Date();
|
||||
Thread.sleep(10);
|
||||
{
|
||||
Patient patient = new Patient();
|
||||
patient.addName().addFamily("EXPIRE");
|
||||
pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
|
||||
}
|
||||
Thread.sleep(10);
|
||||
|
||||
SearchParameterMap params;
|
||||
params = new SearchParameterMap();
|
||||
params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
|
||||
IBundleProvider bundleProvider = myPatientDao.search(params);
|
||||
assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
|
||||
assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
|
||||
|
||||
myDaoConfig.setExpireSearchResults(false);
|
||||
myStaleSearchDeletingSvc.pollForStaleSearches();
|
||||
Thread.sleep(1500);
|
||||
|
||||
assertThat(toUnqualifiedVersionlessIds(bundleProvider), (containsInAnyOrder(pid1, pid2)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchPagesExpiry() throws Exception {
|
||||
IIdType pid1;
|
||||
IIdType pid2;
|
||||
{
|
||||
Patient patient = new Patient();
|
||||
patient.addName().addFamily("EXPIRE");
|
||||
pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
|
||||
}
|
||||
Thread.sleep(10);
|
||||
{
|
||||
Patient patient = new Patient();
|
||||
patient.addName().addFamily("EXPIRE");
|
||||
pid2 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless();
|
||||
}
|
||||
Thread.sleep(10);
|
||||
|
||||
SearchParameterMap params;
|
||||
params = new SearchParameterMap();
|
||||
params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
|
||||
IBundleProvider bundleProvider = myPatientDao.search(params);
|
||||
assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
|
||||
assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
|
||||
|
||||
Thread.sleep(1500);
|
||||
|
||||
myDaoConfig.setExpireSearchResultsAfterMillis(500);
|
||||
myStaleSearchDeletingSvc.pollForStaleSearches();
|
||||
|
||||
assertThat(toUnqualifiedVersionlessIds(bundleProvider), not(containsInAnyOrder(pid1, pid2)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchStringParamReallyLong() {
|
||||
String methodName = "testSearchStringParamReallyLong";
|
||||
|
|
|
|||
|
|
@ -3,13 +3,18 @@ package ca.uhn.fhir.jpa.provider.dstu3;
|
|||
import static org.apache.commons.lang3.StringUtils.isNotBlank;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.servlet.DispatcherType;
|
||||
|
||||
import org.apache.catalina.filters.CorsFilter;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.servlet.FilterHolder;
|
||||
import org.eclipse.jetty.servlet.ServletContextHandler;
|
||||
import org.eclipse.jetty.servlet.ServletHolder;
|
||||
import org.hl7.fhir.dstu3.hapi.validation.DefaultProfileValidationSupport;
|
||||
|
|
@ -36,6 +41,7 @@ import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
|
|||
import ca.uhn.fhir.rest.client.IGenericClient;
|
||||
import ca.uhn.fhir.rest.client.ServerValidationModeEnum;
|
||||
import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
|
||||
import ca.uhn.fhir.rest.server.CORSFilter_;
|
||||
import ca.uhn.fhir.rest.server.FifoMemoryPagingProvider;
|
||||
import ca.uhn.fhir.rest.server.RestfulServer;
|
||||
import ca.uhn.fhir.util.TestUtil;
|
||||
|
|
@ -114,6 +120,15 @@ public abstract class BaseResourceProviderDstu3Test extends BaseJpaDstu3Test {
|
|||
subsServletHolder.setInitParameter(ContextLoader.CONFIG_LOCATION_PARAM, WebsocketDstu3Config.class.getName());
|
||||
proxyHandler.addServlet(subsServletHolder, "/*");
|
||||
|
||||
FilterHolder corsFilterHolder = new FilterHolder();
|
||||
corsFilterHolder.setHeldClass(CorsFilter.class);
|
||||
corsFilterHolder.setInitParameter("cors.allowed.origins", "*");
|
||||
corsFilterHolder.setInitParameter("cors.allowed.methods", "GET,POST,PUT,DELETE,OPTIONS");
|
||||
corsFilterHolder.setInitParameter("cors.allowed.headers", "X-FHIR-Starter,Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers");
|
||||
corsFilterHolder.setInitParameter("cors.exposed.headers", "Location,Content-Location");
|
||||
corsFilterHolder.setInitParameter("cors.support.credentials", "true");
|
||||
corsFilterHolder.setInitParameter("cors.logging.enabled", "true");
|
||||
proxyHandler.addFilter(corsFilterHolder, "/*", EnumSet.of(DispatcherType.REQUEST));
|
||||
|
||||
server.setHandler(proxyHandler);
|
||||
server.start();
|
||||
|
|
|
|||
|
|
@ -19,6 +19,12 @@
|
|||
(e.g. "StructureDefinition.url"). Thanks
|
||||
to David Hay for reporting!
|
||||
</action>
|
||||
<action type="add">
|
||||
Add ability to JPA server for disabling stale search
|
||||
expiry. This is useful if you are deploying the server
|
||||
to a cluster.
|
||||
</action>
|
||||
</release>
|
||||
<release version="2.1" date="2016-11-11">
|
||||
<action type="add">
|
||||
STU3 structure definitions have been updated to the
|
||||
|
|
|
|||
Loading…
Reference in New Issue