honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1698 from jamesagnew/ks-20200203-remove-search-lastused 

Remove searchLastReturned column
This commit is contained in:
Ken Stevens 2020-02-03 16:53:27 -05:00 committed by GitHub
parent d7b45ff65c 2e96d6bc2d
commit a31e90da2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 96 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java
View File

@ -368,7 +368,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStora
Search search = new Search();
search.setDeleted(false);
search.setCreated(new Date());
search.setSearchLastReturned(new Date());
search.setLastUpdated(theSince, theUntil);
search.setUuid(UUID.randomUUID().toString());
search.setResourceType(resourceName);

9
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchDao.java
View File

@ -1,7 +1,6 @@
package ca.uhn.fhir.jpa.dao.data;
import ca.uhn.fhir.jpa.entity.Search;
import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaRepository;
@ -38,16 +37,12 @@ public interface ISearchDao extends JpaRepository<Search, Long> {
@Query("SELECT s FROM Search s LEFT OUTER JOIN FETCH s.myIncludes WHERE s.myUuid = :uuid")
Optional<Search> findByUuidAndFetchIncludes(@Param("uuid") String theUuid);
@Query("SELECT s.myId FROM Search s WHERE (s.mySearchLastReturned < :cutoff) AND (s.myExpiryOrNull IS NULL OR s.myExpiryOrNull < :now)")
Slice<Long> findWhereLastReturnedBefore(@Param("cutoff") Date theCutoff, @Param("now") Date theNow, Pageable thePage);
@Query("SELECT s.myId FROM Search s WHERE (s.myCreated < :cutoff) AND (s.myExpiryOrNull IS NULL OR s.myExpiryOrNull < :now)")
Slice<Long> findWhereCreatedBefore(@Param("cutoff") Date theCutoff, @Param("now") Date theNow, Pageable thePage);
@Query("SELECT s FROM Search s WHERE s.myResourceType = :type AND mySearchQueryStringHash = :hash AND (s.myCreated > :cutoff) AND s.myDeleted = false AND s.myStatus <> 'FAILED'")
Collection<Search> findWithCutoffOrExpiry(@Param("type") String theResourceType, @Param("hash") int theHashCode, @Param("cutoff") Date theCreatedCutoff);
@Modifying
@Query("UPDATE Search s SET s.mySearchLastReturned = :last WHERE s.myId = :pid")
void updateSearchLastReturned(@Param("pid") long thePid, @Param("last") Date theDate);
@Modifying
@Query("UPDATE Search s SET s.myDeleted = :deleted WHERE s.myId = :pid")
void updateDeleted(@Param("pid") Long thePid, @Param("deleted") boolean theDeleted);

18
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java
View File

@ -13,7 +13,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.persistence.*;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.util.*;
@ -43,8 +42,8 @@ import static org.apache.commons.lang3.StringUtils.left;
@Table(name = "HFJ_SEARCH", uniqueConstraints = {
@UniqueConstraint(name = "IDX_SEARCH_UUID", columnNames = "SEARCH_UUID")
}, indexes = {
@Index(name = "IDX_SEARCH_LASTRETURNED", columnList = "SEARCH_LAST_RETURNED"),
@Index(name = "IDX_SEARCH_RESTYPE_HASHS", columnList = "RESOURCE_TYPE,SEARCH_QUERY_STRING_HASH,CREATED")
@Index(name = "IDX_SEARCH_RESTYPE_HASHS", columnList = "RESOURCE_TYPE,SEARCH_QUERY_STRING_HASH,CREATED"),
@Index(name = "IDX_SEARCH_CREATED", columnList = "CREATED")
})
public class Search implements ICachedSearchDetails, Serializable {
@ -90,11 +89,6 @@ public class Search implements ICachedSearchDetails, Serializable {
private Long myResourceId;
@Column(name = "RESOURCE_TYPE", length = 200, nullable = true)
private String myResourceType;
@NotNull
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "SEARCH_LAST_RETURNED", nullable = false, updatable = false)
@OptimisticLock(excluded = true)
private Date mySearchLastReturned;
@Lob()
@Basic(fetch = FetchType.LAZY)
@Column(name = "SEARCH_QUERY_STRING", nullable = true, updatable = false, length = MAX_SEARCH_QUERY_STRING)
@ -261,14 +255,6 @@ public class Search implements ICachedSearchDetails, Serializable {
myResourceType = theResourceType;
}
public Date getSearchLastReturned() {
return mySearchLastReturned;
}
public void setSearchLastReturned(Date theDate) {
mySearchLastReturned = theDate;
}
public String getSearchQueryString() {
return mySearchQueryString;
}

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java
View File

@ -406,8 +406,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
.addIfMatchesType(ServletRequestDetails.class, theRequestDetails);
JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params);
mySearchCacheSvc.updateSearchLastReturned(searchToUse, new Date());
PersistedJpaBundleProvider retVal = new PersistedJpaBundleProvider(theRequestDetails, searchToUse.getUuid(), theCallingDao, mySearchBuilderFactory);
retVal.setCacheHit(true);
populateBundleProvider(retVal);
@ -1142,7 +1140,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
theSearch.setDeleted(false);
theSearch.setUuid(theSearchUuid);
theSearch.setCreated(new Date());
theSearch.setSearchLastReturned(new Date());
theSearch.setTotalCount(null);
theSearch.setNumFound(0);
theSearch.setPreferredPageSize(theParams.getCount());

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/StaleSearchDeletingSvcImpl.java
View File

@ -32,7 +32,7 @@ import org.springframework.transaction.annotation.Transactional;
import javax.annotation.PostConstruct;
import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl.DEFAULT_CUTOFF_SLACK;
import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl.SEARCH_CLEANUP_JOB_INTERVAL_MILLIS;
/**
* Deletes old searches
@ -62,7 +62,7 @@ public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc {
ScheduledJobDefinition jobDetail = new ScheduledJobDefinition();
jobDetail.setId(getClass().getName());
jobDetail.setJobClass(Job.class);
mySchedulerService.scheduleClusteredJob(DEFAULT_CUTOFF_SLACK, jobDetail);
mySchedulerService.scheduleClusteredJob(SEARCH_CLEANUP_JOB_INTERVAL_MILLIS, jobDetail);
}
public static class Job implements HapiJob {

87
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/BaseSearchCacheSvcImpl.java vendored
View File

@ -1,87 +0,0 @@
package ca.uhn.fhir.jpa.search.cache;
/*-
* #%L
* HAPI FHIR JPA Server
* %%
* Copyright (C) 2014 - 2020 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.entity.Search;
import ca.uhn.fhir.jpa.model.sched.HapiJob;
import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
import org.apache.commons.lang3.time.DateUtils;
import org.quartz.JobExecutionContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import javax.annotation.PostConstruct;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public abstract class BaseSearchCacheSvcImpl implements ISearchCacheSvc {
@Autowired
private PlatformTransactionManager myTxManager;
@Autowired
private ISchedulerService mySchedulerService;
private ConcurrentHashMap<Long, Date> myUnsyncedLastUpdated = new ConcurrentHashMap<>();
@Override
public void updateSearchLastReturned(Search theSearch, Date theDate) {
myUnsyncedLastUpdated.put(theSearch.getId(), theDate);
}
@PostConstruct
public void scheduleJob() {
ScheduledJobDefinition jobDetail = new ScheduledJobDefinition();
jobDetail.setId(getClass().getName());
jobDetail.setJobClass(Job.class);
mySchedulerService.scheduleLocalJob(10 * DateUtils.MILLIS_PER_SECOND, jobDetail);
}
@Override
public void flushLastUpdated() {
TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
txTemplate.execute(t -> {
for (Iterator<Map.Entry<Long, Date>> iter = myUnsyncedLastUpdated.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry<Long, Date> next = iter.next();
flushLastUpdated(next.getKey(), next.getValue());
iter.remove();
}
return null;
});
}
protected abstract void flushLastUpdated(Long theSearchId, Date theLastUpdated);
public static class Job implements HapiJob {
@Autowired
private ISearchCacheSvc myTarget;
@Override
public void execute(JobExecutionContext theContext) {
myTarget.flushLastUpdated();
}
}
}

31
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchCacheSvcImpl.java vendored
View File

@ -47,7 +47,7 @@ import java.util.Date;
import java.util.List;
import java.util.Optional;
public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
public class DatabaseSearchCacheSvcImpl implements ISearchCacheSvc {
/*
* Be careful increasing this number! We use the number of params here in a
* DELETE FROM foo WHERE params IN (term,term,term...)
@ -55,7 +55,7 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
*/
public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT = 500;
public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS = 20000;
public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND;
public static final long SEARCH_CLEANUP_JOB_INTERVAL_MILLIS = 10 * DateUtils.MILLIS_PER_SECOND;
private static final Logger ourLog = LoggerFactory.getLogger(DatabaseSearchCacheSvcImpl.class);
private static int ourMaximumResultsToDeleteInOneStatement = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT;
private static int ourMaximumResultsToDeleteInOnePass = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS;
@ -65,7 +65,7 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
* is being reused (because a new client request came in with the same params) right before
* the result is to be deleted
*/
private long myCutoffSlack = DEFAULT_CUTOFF_SLACK;
private long myCutoffSlack = SEARCH_CLEANUP_JOB_INTERVAL_MILLIS;
@Autowired
private ISearchDao mySearchDao;
@ -146,11 +146,6 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
}
@Override
protected void flushLastUpdated(Long theSearchId, Date theLastUpdated) {
mySearchDao.updateSearchLastReturned(theSearchId, theLastUpdated);
}
@Transactional(Transactional.TxType.NEVER)
@Override
public void pollForStaleSearchesAndDeleteThem() {
@ -160,7 +155,7 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis();
if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) {
cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis());
cutoffMillis = cutoffMillis + myDaoConfig.getReuseCachedSearchResultsForMillis();
}
final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack);
@ -168,16 +163,19 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now())));
}
ourLog.debug("Searching for searches which are before {}", cutoff);
// FIXME KHS
ourLog.info("Searching for searches which are before {}", cutoff);
TransactionTemplate tt = new TransactionTemplate(myTxManager);
final Slice<Long> toDelete = tt.execute(theStatus ->
mySearchDao.findWhereLastReturnedBefore(cutoff, new Date(), PageRequest.of(0, 2000))
mySearchDao.findWhereCreatedBefore(cutoff, new Date(), PageRequest.of(0, 2000))
);
assert toDelete != null;
for (final Long nextSearchToDelete : toDelete) {
ourLog.debug("Deleting search with PID {}", nextSearchToDelete);
// FIXME KHS
ourLog.info("Deleting search with PID {}", nextSearchToDelete);
// FIXME KHS do this outside of loop. Make a "deleteWhereCreatedBefore"
tt.execute(t -> {
mySearchDao.updateDeleted(nextSearchToDelete, true);
return null;
@ -191,9 +189,12 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
int count = toDelete.getContent().size();
if (count > 0) {
if (ourLog.isDebugEnabled() || "true".equalsIgnoreCase(System.getProperty("test"))) {
// FIXME KHS
// if (ourLog.isDebugEnabled() || "true".equalsIgnoreCase(System.getProperty("test"))) {
if ( "true".equalsIgnoreCase(System.getProperty("test"))) {
Long total = tt.execute(t -> mySearchDao.count());
ourLog.debug("Deleted {} searches, {} remaining", count, total);
// FIXME KHS
ourLog.info("Deleted {} searches, {} remaining", count, total);
}
}
}
@ -223,7 +224,7 @@ public class DatabaseSearchCacheSvcImpl extends BaseSearchCacheSvcImpl {
// Only delete if we don't have results left in this search
if (resultPids.getNumberOfElements() < max) {
ourLog.debug("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned()));
ourLog.debug("Deleting search {}/{} - Created[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()));
mySearchDao.deleteByPid(searchToDelete.getId());
} else {
ourLog.debug("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid());

14
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/ISearchCacheSvc.java vendored
View File

@ -78,20 +78,6 @@ public interface ISearchCacheSvc {
*/
Collection<Search> findCandidatesForReuse(String theResourceType, String theQueryString, int theQueryStringHash, Date theCreatedAfter);
/**
* Mark a search as having been "last used" at the given time. This method may (and probably should) be implemented
* to work asynchronously in order to avoid hammering the database if the search gets reused many times.
*
* @param theSearch The search
* @param theDate The "last returned" timestamp
*/
void updateSearchLastReturned(Search theSearch, Date theDate);
/**
* This is mostly public for unit tests
*/
void flushLastUpdated();
/**
* This method will be called periodically to delete stale searches. Implementations are not required to do anything
* if they have some other mechanism for expiring stale results other than manually looking for them

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java
View File

@ -64,7 +64,6 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.util.AopTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Transactional;
import javax.persistence.EntityManager;
import java.io.IOException;
@ -409,7 +408,6 @@ public abstract class BaseJpaR4Test extends BaseJpaTest {
}
@Before
@Transactional()
public void beforePurgeDatabase() {
purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc);
}

90
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchPageExpiryTest.java
View File

@ -32,7 +32,7 @@ import javax.annotation.Nullable;
import java.util.Date;
import java.util.concurrent.atomic.AtomicLong;
import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl.DEFAULT_CUTOFF_SLACK;
import static ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl.SEARCH_CLEANUP_JOB_INTERVAL_MILLIS;
import static org.awaitility.Awaitility.await;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.*;
@ -47,7 +47,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
@After()
public void after() {
DatabaseSearchCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchCacheSvc);
staleSearchDeletingSvc.setCutoffSlackForUnitTest(DEFAULT_CUTOFF_SLACK);
staleSearchDeletingSvc.setCutoffSlackForUnitTest(SEARCH_CLEANUP_JOB_INTERVAL_MILLIS);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(null);
}
@ -81,8 +81,11 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
Thread.sleep(10);
myDaoConfig.setExpireSearchResultsAfterMillis(1000L);
myDaoConfig.setReuseCachedSearchResultsForMillis(500L);
long reuseCachedSearchResultsForMillis = 500L;
myDaoConfig.setReuseCachedSearchResultsForMillis(reuseCachedSearchResultsForMillis);
long millisBetweenReuseAndExpire = 800L;
long expireSearchResultsAfterMillis = 1000L;
myDaoConfig.setExpireSearchResultsAfterMillis(expireSearchResultsAfterMillis);
long start = System.currentTimeMillis();
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start);
@ -107,9 +110,9 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
assertEquals(searchUuid1, searchUuid2);
TestUtil.sleepAtLeast(501);
TestUtil.sleepAtLeast(reuseCachedSearchResultsForMillis + 1);
// We're now past 500ms so we shouldn't reuse the search
// We're now past reuseCachedSearchResultsForMillis so we shouldn't reuse the search
final String searchUuid3;
{
@ -124,35 +127,36 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
// Search just got used so it shouldn't be deleted
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start + 500);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start + reuseCachedSearchResultsForMillis);
final AtomicLong search1timestamp = new AtomicLong();
final AtomicLong search2timestamp = new AtomicLong();
final AtomicLong search3timestamp = new AtomicLong();
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
Search search3 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).orElseThrow(()->new InternalErrorException("Search doesn't exist"));
assertNotNull(search3);
Search search1 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).orElseThrow(()->new InternalErrorException("Search doesn't exist"));
assertNotNull(search1);
search1timestamp.set(search1.getCreated().getTime());
Search search2 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid2).orElseThrow(()->new InternalErrorException("Search doesn't exist"));
assertNotNull(search2);
search3timestamp.set(search2.getSearchLastReturned().getTime());
search2timestamp.set(search2.getCreated().getTime());
Search search3 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).orElseThrow(()->new InternalErrorException("Search doesn't exist"));
assertNotNull(search3);
search3timestamp.set(search3.getCreated().getTime());
}
});
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search1timestamp.get() + millisBetweenReuseAndExpire);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3));
}
});
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1));
assertTrue(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).isPresent());
assertTrue(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent());
}
});
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 1100);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search1timestamp.get() + reuseCachedSearchResultsForMillis + expireSearchResultsAfterMillis + 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@ -163,14 +167,12 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
});
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + reuseCachedSearchResultsForMillis + expireSearchResultsAfterMillis + 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
await().until(()-> newTxTemplate().execute(t -> !mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent()));
await().until(()-> newTxTemplate().execute(t -> !mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).isPresent()));
}
@Test
@ -197,7 +199,6 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
params.add(Patient.SP_FAMILY, new StringParam("EXPIRE"));
final IBundleProvider bundleProvider = myPatientDao.search(params);
assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
assertThat(toUnqualifiedVersionlessIds(bundleProvider), containsInAnyOrder(pid1, pid2));
waitForSearchToSave(bundleProvider.getUuid());
final AtomicLong start = new AtomicLong();
@ -213,9 +214,11 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
});
myDaoConfig.setExpireSearchResultsAfterMillis(500);
myDaoConfig.setReuseCachedSearchResultsForMillis(500L);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start.get() + 499);
int expireSearchResultsAfterMillis = 700;
myDaoConfig.setExpireSearchResultsAfterMillis(expireSearchResultsAfterMillis);
long reuseCachedSearchResultsForMillis = 400L;
myDaoConfig.setReuseCachedSearchResultsForMillis(reuseCachedSearchResultsForMillis);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start.get() + expireSearchResultsAfterMillis - 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
txTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
@ -224,7 +227,7 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
});
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start.get() + 600);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(start.get() + expireSearchResultsAfterMillis + reuseCachedSearchResultsForMillis + 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
txTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
@ -251,8 +254,12 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
Thread.sleep(10);
myDaoConfig.setExpireSearchResultsAfterMillis(1000L);
myDaoConfig.setReuseCachedSearchResultsForMillis(500L);
long expireSearchResultsAfterMillis = 1000L;
myDaoConfig.setExpireSearchResultsAfterMillis(expireSearchResultsAfterMillis);
long reuseCachedSearchResultsForMillis = 500L;
myDaoConfig.setReuseCachedSearchResultsForMillis(reuseCachedSearchResultsForMillis);
long start = System.currentTimeMillis();
final String searchUuid1;
@ -278,9 +285,10 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
assertEquals(searchUuid1, searchUuid2);
TestUtil.sleepAtLeast(501);
TestUtil.sleepAtLeast(reuseCachedSearchResultsForMillis + 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
// We're now past 500ms so we shouldn't reuse the search
// We're now past reuseCachedSearchResultsForMillis so we shouldn't reuse the search
final String searchUuid3;
{
@ -293,37 +301,35 @@ public class FhirResourceDaoR4SearchPageExpiryTest extends BaseJpaR4Test {
}
assertNotEquals(searchUuid1, searchUuid3);
// Search just got used so it shouldn't be deleted
waitForSearchToSave(searchUuid3);
// Search hasn't expired yet so it shouldn't be deleted
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
final AtomicLong search1timestamp = new AtomicLong();
final AtomicLong search3timestamp = new AtomicLong();
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
Search search1 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).orElseThrow(()->new InternalErrorException("Search doesn't exist"));
Search search3 = mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).orElseThrow(()->new InternalErrorException("Search doesn't exist"));
assertNotNull(search3);
search1timestamp.set(search1.getCreated().getTime());
search3timestamp.set(search3.getCreated().getTime());
}
});
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 800);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search1timestamp.get() + expireSearchResultsAfterMillis + reuseCachedSearchResultsForMillis + 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
assertNotNull(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3));
}
});
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
assertFalse(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid1).isPresent());
assertTrue(mySearchEntityDao.findByUuidAndFetchIncludes(searchUuid3).isPresent());
}
});
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + 2100);
DatabaseSearchCacheSvcImpl.setNowForUnitTests(search3timestamp.get() + expireSearchResultsAfterMillis + reuseCachedSearchResultsForMillis + 1);
myStaleSearchDeletingSvc.pollForStaleSearchesAndDeleteThem();
newTxTemplate().execute(new TransactionCallbackWithoutResult() {
@Override

16
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java
View File

@ -3005,7 +3005,6 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
.count(5)
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid1 = toSearchUuidFromLinkNext(result1);
Search search1 = newTxTemplate().execute(new TransactionCallback<Search>() {
@ -3014,7 +3013,7 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
return mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException(""));
}
});
Date lastReturned1 = search1.getSearchLastReturned();
Date created1 = search1.getCreated();
Bundle result2 = ourClient
.search()
@ -3023,7 +3022,6 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
.count(5)
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid2 = toSearchUuidFromLinkNext(result2);
Search search2 = newTxTemplate().execute(new TransactionCallback<Search>() {
@ -3032,9 +3030,9 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
return mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException(""));
}
});
Date lastReturned2 = search2.getSearchLastReturned();
Date created2 = search2.getCreated();
assertTrue(lastReturned2.getTime() > lastReturned1.getTime());
assertEquals(created2.getTime(), created1.getTime());
Thread.sleep(1500);
@ -3069,24 +3067,22 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
.forResource("Organization")
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid1 = toSearchUuidFromLinkNext(result1);
Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException("")));
Date lastReturned1 = search1.getSearchLastReturned();
Date created1 = search1.getCreated();
Bundle result2 = ourClient
.search()
.forResource("Organization")
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid2 = toSearchUuidFromLinkNext(result2);
Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException("")));
Date lastReturned2 = search2.getSearchLastReturned();
Date created2 = search2.getCreated();
assertTrue(lastReturned2.getTime() > lastReturned1.getTime());
assertEquals(created2.getTime(), created1.getTime());
assertEquals(uuid1, uuid2);
}

78
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java
View File

@ -14,27 +14,15 @@ import ca.uhn.fhir.model.primitive.UriDt;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.parser.StrictErrorHandler;
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.MethodOutcome;
import ca.uhn.fhir.rest.api.PreferReturnEnum;
import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
import ca.uhn.fhir.rest.api.SummaryEnum;
import ca.uhn.fhir.rest.api.*;
import ca.uhn.fhir.rest.client.api.IClientInterceptor;
import ca.uhn.fhir.rest.client.api.IGenericClient;
import ca.uhn.fhir.rest.client.api.IHttpRequest;
import ca.uhn.fhir.rest.client.api.IHttpResponse;
import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor;
import ca.uhn.fhir.rest.gclient.StringClientParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.StringAndListParam;
import ca.uhn.fhir.rest.param.StringOrListParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.server.exceptions.*;
import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.util.UrlUtil;
@ -46,29 +34,16 @@ import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPatch;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.*;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicNameValuePair;
import org.hamcrest.Matchers;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseBundle;
import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.instance.model.api.*;
import org.hl7.fhir.r4.hapi.validation.FhirInstanceValidator;
import org.hl7.fhir.r4.model.*;
import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.r4.model.Bundle.BundleLinkComponent;
import org.hl7.fhir.r4.model.Bundle.BundleType;
import org.hl7.fhir.r4.model.Bundle.HTTPVerb;
import org.hl7.fhir.r4.model.Bundle.SearchEntryMode;
import org.hl7.fhir.r4.model.Bundle.*;
import org.hl7.fhir.r4.model.Encounter.EncounterLocationComponent;
import org.hl7.fhir.r4.model.Encounter.EncounterStatus;
import org.hl7.fhir.r4.model.Enumerations.AdministrativeGender;
@ -77,11 +52,7 @@ import org.hl7.fhir.r4.model.Observation.ObservationStatus;
import org.hl7.fhir.r4.model.Questionnaire.QuestionnaireItemType;
import org.hl7.fhir.r4.model.Subscription.SubscriptionChannelType;
import org.hl7.fhir.r4.model.Subscription.SubscriptionStatus;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.util.AopTestUtils;
import org.springframework.transaction.TransactionStatus;
@ -96,28 +67,14 @@ import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.*;
import java.util.stream.Collectors;
import static ca.uhn.fhir.jpa.util.TestUtil.sleepAtLeast;
import static ca.uhn.fhir.jpa.util.TestUtil.sleepOneClick;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
@SuppressWarnings("Duplicates")
public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
@ -4115,11 +4072,10 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
.count(5)
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid1 = toSearchUuidFromLinkNext(result1);
Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException("")));
Date lastReturned1 = search1.getSearchLastReturned();
Date created1 = search1.getCreated();
Bundle result2 = ourClient
.search()
@ -4128,13 +4084,12 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
.count(5)
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid2 = toSearchUuidFromLinkNext(result2);
Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException("")));
Date lastReturned2 = search2.getSearchLastReturned();
Date created2 = search2.getCreated();
assertTrue(lastReturned2.getTime() > lastReturned1.getTime());
assertEquals(created2.getTime(), created1.getTime());
Thread.sleep(1500);
@ -4145,7 +4100,6 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
.count(5)
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
String uuid3 = toSearchUuidFromLinkNext(result3);
@ -4170,11 +4124,10 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
.forResource("Organization")
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid1 = toSearchUuidFromLinkNext(result1);
Search search1 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid1).orElseThrow(() -> new InternalErrorException("")));
Date lastReturned1 = search1.getSearchLastReturned();
Date created1 = search1.getCreated();
sleepOneClick();
@ -4183,13 +4136,12 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
.forResource("Organization")
.returnBundle(Bundle.class)
.execute();
mySearchCacheSvc.flushLastUpdated();
final String uuid2 = toSearchUuidFromLinkNext(result2);
Search search2 = newTxTemplate().execute(theStatus -> mySearchEntityDao.findByUuidAndFetchIncludes(uuid2).orElseThrow(() -> new InternalErrorException("")));
Date lastReturned2 = search2.getSearchLastReturned();
Date created2 = search2.getCreated();
assertTrue(lastReturned2.getTime() > lastReturned1.getTime());
assertEquals(created2.getTime(), created1.getTime());
assertEquals(uuid1, uuid2);
}

8
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/StaleSearchDeletingSvcR4Test.java
View File

@ -9,12 +9,10 @@ import ca.uhn.fhir.jpa.entity.SearchResult;
import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
import ca.uhn.fhir.jpa.model.entity.ResourceTable;
import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
import ca.uhn.fhir.jpa.search.StaleSearchDeletingSvcImpl;
import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl;
import ca.uhn.fhir.rest.gclient.IClientExecutable;
import ca.uhn.fhir.rest.gclient.IQuery;
import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
import ca.uhn.fhir.test.utilities.UnregisterScheduledProcessor;
import ca.uhn.fhir.util.TestUtil;
import org.apache.commons.lang3.time.DateUtils;
import org.hl7.fhir.r4.model.Bundle;
@ -25,7 +23,6 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.util.AopTestUtils;
import java.util.Date;
@ -50,7 +47,7 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test {
public void after() throws Exception {
super.after();
DatabaseSearchCacheSvcImpl staleSearchDeletingSvc = AopTestUtils.getTargetObject(mySearchCacheSvc);
staleSearchDeletingSvc.setCutoffSlackForUnitTest(DatabaseSearchCacheSvcImpl.DEFAULT_CUTOFF_SLACK);
staleSearchDeletingSvc.setCutoffSlackForUnitTest(DatabaseSearchCacheSvcImpl.SEARCH_CLEANUP_JOB_INTERVAL_MILLIS);
DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteForUnitTest(DatabaseSearchCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT);
DatabaseSearchCacheSvcImpl.setMaximumResultsToDeleteInOnePassForUnitTest(DatabaseSearchCacheSvcImpl.DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS);
}
@ -121,7 +118,6 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test {
search.setCreated(DateUtils.addDays(new Date(), -10000));
search.setSearchType(SearchTypeEnum.SEARCH);
search.setResourceType("Patient");
search.setSearchLastReturned(DateUtils.addDays(new Date(), -10000));
search = mySearchEntityDao.save(search);
for (int i = 0; i < 15; i++) {
@ -163,7 +159,6 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test {
search.setCreated(DateUtils.addDays(new Date(), -10000));
search.setSearchType(SearchTypeEnum.SEARCH);
search.setResourceType("Patient");
search.setSearchLastReturned(DateUtils.addDays(new Date(), -10000));
mySearchEntityDao.save(search);
});
@ -190,7 +185,6 @@ public class StaleSearchDeletingSvcR4Test extends BaseResourceProviderR4Test {
search.setCreated(DateUtils.addDays(new Date(), -10000));
search.setSearchType(SearchTypeEnum.SEARCH);
search.setResourceType("Patient");
search.setSearchLastReturned(DateUtils.addDays(new Date(), -10000));
search = mySearchEntityDao.save(search);
});

4
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
View File

@ -61,6 +61,10 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
protected void init420() { // 20191015 - present
Builder version = forVersion(VersionEnum.V4_2_0);
Builder.BuilderWithTableName searchTable = version.onTable("HFJ_SEARCH");
searchTable.dropIndex("20200203.1", "IDX_SEARCH_LASTRETURNED");
searchTable.dropColumn("20200203.2", "SEARCH_LAST_RETURNED");
searchTable.addIndex("20200203.3", "IDX_SEARCH_CREATED").unique(false).withColumns("CREATED");
}
protected void init410() { // 20190815 - 20191014