honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

partition cache (#5217) 

* Convert PartitionLookupSvcImpl to use MemoryCacheService

* Convert PartitionLookupSvcImpl to use MemoryCacheService

* add test

* add test

* add test

* fix tests
This commit is contained in:
Ken Stevens 2023-08-18 18:08:54 -04:00 committed by GitHub
parent 1e45506526
commit ab2a86b0d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 91 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/partition/IPartitionLookupSvc.java
View File

@ -44,7 +44,7 @@ public interface IPartitionLookupSvc {
*/
PartitionEntity getPartitionById(Integer theId) throws ResourceNotFoundException;
void clearCaches();
void invalidateCaches();
/**
* Will generate a random unused partition ID. Validates that no partition with that ID exists before returning.

45
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/partition/PartitionLookupSvcImpl.java
View File

@ -29,18 +29,14 @@ import ca.uhn.fhir.jpa.dao.data.IPartitionDao;
import ca.uhn.fhir.jpa.entity.PartitionEntity;
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
import ca.uhn.fhir.jpa.model.util.JpaConstants;
import ca.uhn.fhir.jpa.util.MemoryCacheService;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.sl.cache.CacheFactory;
import ca.uhn.fhir.sl.cache.CacheLoader;
import ca.uhn.fhir.sl.cache.LoadingCache;
import ca.uhn.fhir.util.ICallable;
import org.apache.commons.lang3.Validate;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -51,7 +47,6 @@ import org.springframework.transaction.support.TransactionTemplate;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
@ -73,8 +68,8 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
@Autowired
private IPartitionDao myPartitionDao;
private LoadingCache<String, PartitionEntity> myNameToPartitionCache;
private LoadingCache<Integer, PartitionEntity> myIdToPartitionCache;
@Autowired
private MemoryCacheService myMemoryCacheService;
@Autowired
private FhirContext myFhirCtx;
@ -94,8 +89,6 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
@Override
@PostConstruct
public void start() {
myNameToPartitionCache = CacheFactory.build(TimeUnit.MINUTES.toMillis(1), new NameToPartitionCacheLoader());
myIdToPartitionCache = CacheFactory.build(TimeUnit.MINUTES.toMillis(1), new IdToPartitionCacheLoader());
myTxTemplate = new TransactionTemplate(myTxManager);
}
@ -106,7 +99,8 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
if (JpaConstants.DEFAULT_PARTITION_NAME.equals(theName)) {
return null;
}
return myNameToPartitionCache.get(theName);
return myMemoryCacheService.get(
MemoryCacheService.CacheEnum.NAME_TO_PARTITION, theName, this::lookupPartitionByName);
}
@Override
@ -119,13 +113,14 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
&& myPartitionSettings.getDefaultPartitionId().equals(thePartitionId)) {
return new PartitionEntity().setId(thePartitionId).setName(JpaConstants.DEFAULT_PARTITION_NAME);
}
return myIdToPartitionCache.get(thePartitionId);
return myMemoryCacheService.get(
MemoryCacheService.CacheEnum.ID_TO_PARTITION, thePartitionId, this::lookupPartitionById);
}
@Override
public void clearCaches() {
myNameToPartitionCache.invalidateAll();
myIdToPartitionCache.invalidateAll();
public void invalidateCaches() {
myMemoryCacheService.invalidateCaches(
MemoryCacheService.CacheEnum.NAME_TO_PARTITION, MemoryCacheService.CacheEnum.ID_TO_PARTITION);
}
/**
@ -188,7 +183,7 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
existingPartition.setName(thePartition.getName());
existingPartition.setDescription(thePartition.getDescription());
myPartitionDao.save(existingPartition);
clearCaches();
invalidateCaches();
return existingPartition;
}
@ -208,7 +203,7 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
myPartitionDao.delete(partition.get());
clearCaches();
invalidateCaches();
}
@Override
@ -292,22 +287,6 @@ public class PartitionLookupSvcImpl implements IPartitionLookupSvc {
return myTxTemplate.execute(tx -> theCallable.call());
}
private class NameToPartitionCacheLoader implements @NonNull CacheLoader<String, PartitionEntity> {
@Nullable
@Override
public PartitionEntity load(@NonNull String theName) {
return lookupPartitionByName(theName);
}
}
private class IdToPartitionCacheLoader implements @NonNull CacheLoader<Integer, PartitionEntity> {
@Nullable
@Override
public PartitionEntity load(@NonNull Integer theId) {
return lookupPartitionById(theId);
}
}
public static void validatePartitionIdSupplied(FhirContext theFhirContext, Integer thePartitionId) {
if (thePartitionId == null) {
String msg =

6
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/imprt2/ConsumeFilesStepR4Test.java
View File

@ -146,7 +146,11 @@ public class ConsumeFilesStepR4Test extends BasePartitioningR4Test {
// Validate
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
if (partitionEnabled) {
assertEquals(8, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
} else {
assertEquals(7, myCaptureQueriesListener.countSelectQueriesForCurrentThread());
}
assertEquals(2, myCaptureQueriesListener.countInsertQueriesForCurrentThread());
assertEquals(4, myCaptureQueriesListener.countUpdateQueriesForCurrentThread());
assertEquals(0, myCaptureQueriesListener.countDeleteQueries());

58
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/expunge/ExpungeEverythingServiceTest.java Normal file
View File

@ -0,0 +1,58 @@
package ca.uhn.fhir.jpa.dao.expunge;
import ca.uhn.fhir.jpa.entity.PartitionEntity;
import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.jpa.test.BaseJpaR4Test;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import org.hl7.fhir.instance.model.api.IIdType;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
class ExpungeEverythingServiceTest extends BaseJpaR4Test {
@Autowired
private ExpungeEverythingService myExpungeEverythingService;
@Autowired
private IPartitionLookupSvc myPartitionLookupSvc;
@Test
public void testExpungeEverythingInvalidatesPartitionCache() {
// Setup
IIdType p1 = createPatient(withActiveTrue());
PartitionEntity partition = new PartitionEntity();
partition.setId(123);
partition.setName("PART");
myPartitionLookupSvc.createPartition(partition, mySrd);
// validate precondition
assertEquals(1, myPatientDao.search(SearchParameterMap.newSynchronous()).size());
assertEquals("PART", myPartitionLookupSvc.getPartitionById(123).getName());
assertEquals(123, myPartitionLookupSvc.getPartitionByName("PART").getId());
// execute
myExpungeEverythingService.expungeEverything(mySrd);
// Validate
assertThat(myPartitionLookupSvc.listPartitions(), hasSize(0));
try {
myPartitionLookupSvc.getPartitionById(123);
fail();
} catch (ResourceNotFoundException e) {
assertEquals("No partition exists with ID 123", e.getMessage());
}
try {
myPartitionLookupSvc.getPartitionByName("PART");
fail();
} catch (ResourceNotFoundException e) {
assertEquals("Partition name \"PART\" is not valid", e.getMessage());
}
assertDoesntExist(p1);
}
}

3
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/partition/PartitionLookupSvcImplTest.java
View File

@ -5,6 +5,7 @@ import ca.uhn.fhir.interceptor.api.IInterceptorService;
import ca.uhn.fhir.jpa.cache.ResourceChangeListenerRegistryInterceptor;
import ca.uhn.fhir.jpa.dao.data.IPartitionDao;
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
import ca.uhn.fhir.jpa.util.MemoryCacheService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
@ -36,6 +37,8 @@ class PartitionLookupSvcImplTest {
private FhirContext myFhirCtx;
@MockBean
private PlatformTransactionManager myTxManager;
@MockBean
private MemoryCacheService myMemoryCacheService;
@Configuration
static class SpringContext {

3
hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java
View File

@ -520,6 +520,8 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
@Autowired
protected IMdmLinkJpaRepository myMdmLinkDao;
@Autowired
protected IMdmLinkJpaRepository myMdmLinkHistoryDao;
@Autowired
private IValidationSupport myJpaValidationSupportChainR4;
private PerformanceTracingLoggingInterceptor myPerformanceTracingLoggingInterceptor;
@Autowired
@ -588,6 +590,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil
@AfterEach
public void afterPurgeDatabase() {
runInTransaction(() -> {
myMdmLinkHistoryDao.deleteAll();
myMdmLinkDao.deleteAll();
});
purgeDatabase(myStorageSettings, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper);

3
hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java
View File

@ -366,9 +366,6 @@ public abstract class BaseJpaTest extends BaseTest {
if (myCaptureQueriesListener != null) {
myCaptureQueriesListener.clear();
}
if (myPartitionConfigSvc != null) {
myPartitionConfigSvc.clearCaches();
}
if (myMemoryCacheService != null) {
myMemoryCacheService.invalidateAllCaches();
}

10
hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/util/MemoryCacheService.java
View File

@ -183,6 +183,12 @@ public class MemoryCacheService {
return getCache(theCache).estimatedSize();
}
public void invalidateCaches(CacheEnum... theCaches) {
for (CacheEnum next : theCaches) {
getCache(next).invalidateAll();
}
}
public enum CacheEnum {
TAG_DEFINITION(TagDefinitionCacheKey.class),
RESOURCE_LOOKUP(String.class),
@ -196,7 +202,9 @@ public class MemoryCacheService {
MATCH_URL(String.class),
CONCEPT_TRANSLATION_REVERSE(TranslationQuery.class),
RESOURCE_CONDITIONAL_CREATE_VERSION(Long.class),
HISTORY_COUNT(HistoryCountKey.class);
HISTORY_COUNT(HistoryCountKey.class),
NAME_TO_PARTITION(String.class),
ID_TO_PARTITION(Integer.class);
public Class<?> getKeyType() {
return myKeyType;