honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Shelfving

This commit is contained in:
leif stawnyczy 2024-09-13 11:15:39 -04:00
parent ab2ba05fb5
commit dd9dcfd501
4 changed files with 677 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java
View File

@ -436,7 +436,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc, IHas
return retVal;
}
private boolean isJobsExecuting() {
public boolean isJobsExecuting() {
cleanseEndedJobs();
return !myJobExecutions.isEmpty();

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/api/ITermDeferredStorageSvc.java
View File

@ -78,6 +78,8 @@ public interface ITermDeferredStorageSvc {
void logQueueForUnitTest();
boolean isJobsExecuting();
/**
* Only to be used from tests - Disallow test timeouts on deferred tasks
*/

672
hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4Test.java
View File

@ -1,5 +1,9 @@
package ca.uhn.fhir.jpa.term;
import ca.uhn.fhir.batch2.api.IJobCoordinator;
import ca.uhn.fhir.batch2.jobs.reindex.ReindexAppCtx;
import ca.uhn.fhir.batch2.jobs.reindex.ReindexJobParameters;
import ca.uhn.fhir.batch2.model.JobInstanceStartRequest;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.context.support.ConceptValidationOptions;
@ -9,6 +13,7 @@ import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse;
import ca.uhn.fhir.jpa.entity.TermCodeSystem;
import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
import ca.uhn.fhir.jpa.entity.TermConcept;
@ -25,9 +30,13 @@ import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
import ca.uhn.fhir.jpa.util.SqlQuery;
import ca.uhn.fhir.jpa.util.ValueSetTestUtil;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.util.BundleBuilder;
import com.google.common.collect.Lists;
import jakarta.annotation.Nonnull;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -45,16 +54,22 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Pageable;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static ca.uhn.fhir.batch2.jobs.termcodesystem.TermCodeSystemJobConfig.TERM_CODE_SYSTEM_DELETE_JOB_NAME;
import static ca.uhn.fhir.batch2.jobs.termcodesystem.TermCodeSystemJobConfig.TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME;
import static ca.uhn.fhir.util.HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
@ -2081,7 +2096,662 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test implements IValueSet
outcome = myValueSetDao.validateCode(vs.getUrlElement(), null, new StringType("A"), cs.getUrlElement(), null, null, null, mySrd);
assertEquals(false, outcome.isOk());
assertThat(outcome.getMessage()).contains("Code validation occurred using a ValueSet expansion that was pre-calculated");
}
@Test
public void expandValuesetAfterReindex() {
// setup
IParser parser = myFhirContext.newJsonParser();
CodeSystem cs;
ValueSet vs;
String csStr;
{
String vsStr = """
{
"resourceType": "ValueSet",
"id": "0447bffa-01fa-4405-828a-96192e74a5d8",
"meta": {
"versionId": "2",
"lastUpdated": "2024-04-09T15:06:24.025+00:00",
"source": "#f4491e490a6a2900"
},
"url": "https://health.gov.on.ca/idms/fhir/ValueSet/IDMS-Submission-Types",
"version": "1.0.0",
"name": "IDMS-SUBMISSION-TYPES",
"title": "IDMS Submission Types",
"status": "active",
"experimental": false,
"date": "2023-09-28",
"publisher": "IDMS",
"description": "List of Submission Types",
"compose": {
"include": [
{
"system": "https://health.gov.on.ca/idms/fhir/CodeSystem/Internal-Submission-Types"
}
]
}
}
""";
vs = parser.parseResource(ValueSet.class, vsStr);
csStr = """
{
"resourceType": "CodeSystem",
"id": "d9acd5b8-9533-4fa1-bb70-b4380957a8c3",
"meta": {
"versionId": "14",
"lastUpdated": "2024-06-03T17:49:56.580+00:00",
"source": "#261a82258b0978a8"
},
"url": "https://health.gov.on.ca/idms/fhir/CodeSystem/Internal-Submission-Types",
"version": "1.0.0",
"name": "IDMS-Internal-Submission-Types",
"status": "active",
"date": "2023-09-07",
"publisher": "IDMS",
"description": "This contains a lists of codes Submission Type Codes.",
"content": "complete",
"concept": [
{
"code": "SUB-BRAND-PRODUCT",
"display": "New Brand Product (Non-New Chemical Entity)"
},
{
"code": "SUB-CLASSIFICATION",
"display": "Classification Change"
},
{
"code": "SUB-CLINICIAN-LED-SUBMISSIONS",
"display": "Clinician-Led Submissions"
},
{
"code": "SUB-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-DIN-CHANGE",
"display": "Drug Identification Number (DIN) Change"
},
{
"code": "SUB-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-DRUG-PRODUCT-NAME",
"display": "Drug Product Name Change"
},
{
"code": "SUB-FORMULATION",
"display": "Formulation Change"
},
{
"code": "SUB-GENERIC-LINE",
"display": "Generic Line Extension"
},
{
"code": "SUB-GENERIC-PRODUCT",
"display": "New Generic Product"
},
{
"code": "SUB-LINE-EXTENSION",
"display": "Line Extension"
},
{
"code": "SUB-NEW-INDICATION",
"display": "New Indication"
},
{
"code": "SUB-NEW-NATURAL",
"display": "New Natural Health Product"
},
{
"code": "SUB-NEW-OVER-COUNTER",
"display": "New Over the Counter (OTC) Product"
},
{
"code": "SUB-NEW-CHEMICAL",
"display": "New Chemical Entity"
},
{
"code": "SUB-NEW-NUTRITIONAL",
"display": "New Nutrition Product"
},
{
"code": "SUB-NUTRITIONAL-LINE",
"display": "Line Extension"
},
{
"code": "SUB-NEW-BIOSIMILAR",
"display": "New Biosimilar"
},
{
"code": "SUB-BIOSIMILAR-LINE",
"display": "Line Extension"
},
{
"code": "SUB-NEW-DIABETIC",
"display": "New Diabetic Testing Agent (DTA)"
},
{
"code": "SUB-NEW-GLUCOUSE",
"display": "New Flash Glucose Monitoring (FGM) Product"
},
{
"code": "SUB-NON-FORMULARY-SINGLE",
"display": "Non-Formulary Single Source"
},
{
"code": "SUB-NON-FORMULARY-MULTIPLE",
"display": "Non-Formulary Multiple Source "
},
{
"code": "SUB-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-PRODUCT-MONOGRAPH",
"display": "Product Monograph Change"
},
{
"code": "SUB-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-PRODUCT-CHANGE",
"display": "Product Change"
},
{
"code": "SUB-SPECIAL-PROJECTS",
"display": "Special Projects"
},
{
"code": "SUB-TEMPORARY-BENEFITS",
"display": "Temporary Benefits"
},
{
"code": "SUB-VALVED",
"display": "New Valved Holding Chamber (VHC)"
},
{
"code": "SUB-REDESIGNATION-BENEFITS",
"display": "Re-Designation of Benefits"
},
{
"code": "SUB-MANUFACTURER-INQUIRIES",
"display": "Manufacturer Inquiries"
},
{
"code": "SUB-CHALLENGES",
"display": "Challenges"
},
{
"code": "SUB-NEW-LINE",
"display": "Line Extension"
},
{
"code": "SUB-MULTI-DIN-CHANGE",
"display": "Drug Identification Number (DIN) Change"
},
{
"code": "SUB-MULTI-DRUG-PRODUCT-NAME",
"display": "Drug Product Name Change"
},
{
"code": "SUB-MULTI-PRODUCT-MONOGRAPH",
"display": "Product Monograph Change"
},
{
"code": "SUB-MULTI-FORMULATION",
"display": "Formulation Change"
},
{
"code": "SUB-MULTI-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-MULTI-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-MULTI-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-MULTI-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-MULTI-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-MULTI-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-MULTI-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-BIO-DIN-CHANGE",
"display": "Drug Identification Number (DIN) Change"
},
{
"code": "SUB-BIO-DRUG-PRODUCT-NAME",
"display": "Drug Product Name Change"
},
{
"code": "SUB-BIO-PRODUCT-MONOGRAPH",
"display": "Product Monograph Change"
},
{
"code": "SUB-BIO-FORMULATION",
"display": "Formulation Change"
},
{
"code": "SUB-BIO-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-BIO-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-BIO-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-BIO-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-BIO-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-BIO-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-BIO-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-OTC-DIN-CHANGE",
"display": "Drug Identification Number (DIN) Change"
},
{
"code": "SUB-OTC-DRUG-PRODUCT-NAME",
"display": "Drug Product Name Change"
},
{
"code": "SUB-OTC-PRODUCT-MONOGRAPH",
"display": "Product Monograph Change"
},
{
"code": "SUB-OTC-FORMULATION",
"display": "Formulation Change"
},
{
"code": "SUB-OTC-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-OTC-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-OTC-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-OTC-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-OTC-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-OTC-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-OTC-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-DIABETIC-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-DIABETIC-PRODUCT-CHANGE",
"display": "Product Change"
},
{
"code": "SUB-DIABETIC-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-DIABETIC-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-DIABETIC-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-DIABETIC-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-DIABETIC-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-DIABETIC-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-DIABETIC-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-GLUCOSE-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-GLUCOSE-PRODUCT-CHANGE",
"display": "Product Change"
},
{
"code": "SUB-GLUCOSE-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-GLUCOSE-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-GLUCOSE-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-GLUCOSE-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-GLUCOSE-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-GLUCOSE-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-GLUCOSE-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-VALVED-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-VALVED-PRODUCT-CHANGE",
"display": "Product Change"
},
{
"code": "SUB-VALVED-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-VALVED-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-VALVED-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-VALVED-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-VALVED-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-VALVED-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-VALVED-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-NATURAL-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-NATURAL-PRODUCT-CHANGE",
"display": "Product Change"
},
{
"code": "SUB-NATURAL-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-NATURAL-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-NATURAL-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-NATURAL-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-NATURAL-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-NATURAL-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-NATURAL-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-OTHER-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-OTHER-PRODUCT-CHANGE",
"display": "Product Change"
},
{
"code": "SUB-OTHER-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-OTHER-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-OTHER-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-OTHER-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-OTHER-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-OTHER-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-OTHER-OTHER-TYPE-IN",
"display": "Other"
},
{
"code": "SUB-NUTRITIONAL-PRODUCT-NAME",
"display": "Product Name Change"
},
{
"code": "SUB-NUTRITIONAL-FORMULATION",
"display": "Formulation Change"
},
{
"code": "SUB-NUTRITIONAL-DRUG-REGULATION",
"display": "Change in Food and Drug Regulation Classification"
},
{
"code": "SUB-ADVERTISING-POLICY",
"display": "Change in Advertising Policy"
},
{
"code": "SUB-NUTRITIONAL-OWNERSHIP",
"display": "Ownership Change"
},
{
"code": "SUB-NUTRITIONAL-DISTRIBUTOR",
"display": "Distributor Change"
},
{
"code": "SUB-NUTRITIONAL-ORG-NAME",
"display": "Organization Name Change"
},
{
"code": "SUB-NUTRITIONAL-PRODUCT-DISCONTINUE",
"display": "Product Discontinuation"
},
{
"code": "SUB-NUTRITIONAL-DELISTING",
"display": "Delisting"
},
{
"code": "SUB-NUTRITIONAL-PRICE-CHANGE",
"display": "Price Change"
},
{
"code": "SUB-NUTRITIONAL-OTHER-TYPE-IN",
"display": "Other"
}
]
}
""";
cs = parser.parseResource(CodeSystem.class, csStr);
}
RequestDetails rq = new SystemRequestDetails();
int csToCreate = 1;
// create a bunch of code systems
myValueSetDao.update(vs, rq);
String csUrl = cs.getUrl();
BundleBuilder builder = new BundleBuilder(myFhirContext);
// while (cs.getConcept().size() > 90) {
// cs.getConcept().remove(1);
// }
for (int i = 0; i < csToCreate; i++) {
if (i > 0) {
cs = parser.parseResource(CodeSystem.class, csStr);
cs.setId("cs" + i);
cs.setUrl(csUrl + "-" + i);
}
builder.addTransactionUpdateEntry(cs);
// myCodeSystemDao.update(cs, rq);
}
builder.setType("transaction");
Bundle bundle = (Bundle) builder.getBundle();
mySystemDao.transaction(rq, bundle);
System.out.println("XXXX " + myDeferredStorageSvc.isStorageQueueEmpty());
myDeferredStorageSvc.logQueueForUnitTest();
AtomicInteger counter = new AtomicInteger();
await()
.atMost(10, TimeUnit.SECONDS)
.pollDelay(500, TimeUnit.MILLISECONDS)
.until(() -> {
boolean isQueueEmpty = myDeferredStorageSvc.isStorageQueueEmpty(false);
if (!isQueueEmpty) {
myDeferredStorageSvc.saveAllDeferred();
}
return isQueueEmpty;
});
if (myDeferredStorageSvc.isJobsExecuting()) {
System.out.println("running jobs");
myDeferredStorageSvc.saveAllDeferred();
myBatch2JobHelper.awaitJobCompletion(TERM_CODE_SYSTEM_VERSION_DELETE_JOB_NAME);
myBatch2JobHelper.awaitJobCompletion(TERM_CODE_SYSTEM_DELETE_JOB_NAME);
}
System.out.println("XXX " + myDeferredStorageSvc.isStorageQueueEmpty() + " " + counter.get());
myDeferredStorageSvc.logQueueForUnitTest();
myDeferredStorageSvc.toString();
// myDeferredStorageSvc.saveAllDeferred();
// perform the reindex
ReindexJobParameters params = new ReindexJobParameters();
params.addUrl("CodeSystem?");
JobInstanceStartRequest startRequest = new JobInstanceStartRequest();
startRequest.setJobDefinitionId(ReindexAppCtx.JOB_REINDEX);
startRequest.setParameters(params);
System.out.println("xxxx starting reindex");
Batch2JobStartResponse response = myJobCoordinator.startInstance(rq, startRequest);
myBatch2JobHelper.awaitJobCompletion(response);
System.out.println("XXXX reindex done " + myDeferredStorageSvc.isStorageQueueEmpty());
// get the codes
SearchParameterMap spMap = new SearchParameterMap();
spMap.setLoadSynchronous(true);
IBundleProvider bp = myCodeSystemDao.search(spMap, rq);
// do value expansion
ValueSetExpansionOptions options = new ValueSetExpansionOptions();
options.setCount(200);
ValueSet expanded = myValueSetDao.expand(vs, options);
assertNotNull(expanded);
HashSet<String> all = new HashSet<>();
for (CodeSystem.ConceptDefinitionComponent set : cs.getConcept()) {
all.add(set.getCode());
}
for (ValueSet.ValueSetExpansionContainsComponent v : expanded.getExpansion().getContains()) {
all.remove(v.getCode());
}
assertTrue(all.isEmpty(), String.join(", ", all));
assertEquals(cs.getConcept().size(), expanded.getExpansion().getTotal());
}
@Autowired
private IJobCoordinator myJobCoordinator;
@Autowired
private ITermDeferredStorageSvc myDeferredStorageSvc;
}

4
hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexAppCtx.java
View File

@ -59,7 +59,9 @@ public class ReindexAppCtx {
"Load IDs of resources to reindex",
ResourceIdListWorkChunkJson.class,
reindexLoadIdsStep(theBatch2DaoSvc))
.addLastStep("reindex", "Perform the resource reindex", reindexStep())
.addLastStep("reindex",
"Perform the resource reindex",
reindexStep())
.build();
}