honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement both a copyright property and a copyright filter for LOINC. #1451

This commit is contained in:
Diederik Muylwyk 2019-09-13 16:18:51 -04:00
parent 678d58ab90
commit 9b1af6b207
2 changed files with 334 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseHapiTerminologySvcImpl.java
View File

@ -199,6 +199,20 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
}
}
private void addCopyrightFilter3rdParty(BooleanJunction<?> bool) {
// FIXME: DM 2019-09-13 - This feels hacky but it works until we have a better way for filtering TermConcept based on TermConceptProperty.
Term term = new Term(TermConceptPropertyFieldBridge.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE", ".*");
RegexpQuery query = new RegexpQuery(term);
bool.must(query);
}
private void addCopyrightFilterLoinc(BooleanJunction<?> bool) {
// FIXME: DM 2019-09-13 - This feels hacky but it works until we have a better way for filtering TermConcept based on TermConceptProperty.
Term term = new Term(TermConceptPropertyFieldBridge.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE", ".*");
RegexpQuery query = new RegexpQuery(term);
bool.must(query).not();
}
private void addDisplayFilterExact(QueryBuilder qb, BooleanJunction<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
bool.must(qb.phrase().onField("myDisplay").sentence(nextFilter.getValue()).createQuery());
}
@ -728,74 +742,9 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
*/
if (theInclude.getFilter().size() > 0) {
for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) {
if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) {
continue;
handleFilter(system, qb, bool, nextFilter);
}
if (isBlank(nextFilter.getValue()) || nextFilter.getOp() == null || isBlank(nextFilter.getProperty())) {
throw new InvalidRequestException("Invalid filter, must have fields populated: property op value");
}
if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
addDisplayFilterExact(qb, bool, nextFilter);
} else if ("display".equals(nextFilter.getProperty()) && nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
if (nextFilter.getValue().trim().contains(" ")) {
addDisplayFilterExact(qb, bool, nextFilter);
} else {
addDisplayFilterInexact(qb, bool, nextFilter);
}
} else if (nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) {
TermConcept code = findCode(system, nextFilter.getValue())
.orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + system + "}" + nextFilter.getValue()));
if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
bool.must(qb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery());
} else {
throw new InvalidRequestException("Don't know how to handle op=" + nextFilter.getOp() + " on property " + nextFilter.getProperty());
}
} else {
if (nextFilter.getOp() == ValueSet.FilterOperator.REGEX) {
/*
* We treat the regex filter as a match on the regex
* anywhere in the property string. The spec does not
* say whether or not this is the right behaviour, but
* there are examples that seem to suggest that it is.
*/
String value = nextFilter.getValue();
if (value.endsWith("$")) {
value = value.substring(0, value.length() - 1);
} else if (!value.endsWith(".*")) {
value = value + ".*";
}
if (!value.startsWith("^") && !value.startsWith(".*")) {
value = ".*" + value;
} else if (value.startsWith("^")) {
value = value.substring(1);
}
Term term = new Term(TermConceptPropertyFieldBridge.CONCEPT_FIELD_PROPERTY_PREFIX + nextFilter.getProperty(), value);
RegexpQuery query = new RegexpQuery(term);
bool.must(query);
} else {
String value = nextFilter.getValue();
Term term = new Term(TermConceptPropertyFieldBridge.CONCEPT_FIELD_PROPERTY_PREFIX + nextFilter.getProperty(), value);
bool.must(new TermsQuery(term));
}
}
}
}
Query luceneQuery = bool.createQuery();
@ -925,6 +874,106 @@ public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc,
}
private void handleFilter(String theSystem, QueryBuilder theQb, BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent nextFilter) {
if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) {
return;
}
if (isBlank(nextFilter.getValue()) || nextFilter.getOp() == null || isBlank(nextFilter.getProperty())) {
throw new InvalidRequestException("Invalid filter, must have fields populated: property op value");
}
if (nextFilter.getProperty().equals("display:exact") || nextFilter.getProperty().equals("display")) {
handleDisplayFilter(theQb, theBool, nextFilter);
} else if (nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) {
handleConceptAndCodeFilter(theSystem, theQb, theBool, nextFilter);
} else if (nextFilter.getProperty().equals("copyright")) {
handleLoincCopyrightFilter(theQb, theBool, nextFilter);
} else {
handleRegexFilter(theBool, nextFilter);
}
}
private void handleDisplayFilter(QueryBuilder theQb, BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent nextFilter) {
if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
addDisplayFilterExact(theQb, theBool, nextFilter);
} else if (nextFilter.getProperty().equals("display") && nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
if (nextFilter.getValue().trim().contains(" ")) {
addDisplayFilterExact(theQb, theBool, nextFilter);
} else {
addDisplayFilterInexact(theQb, theBool, nextFilter);
}
}
}
private void handleConceptAndCodeFilter(String theSystem, QueryBuilder theQb, BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent nextFilter) {
TermConcept code = findCode(theSystem, nextFilter.getValue())
.orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + theSystem + "}" + nextFilter.getValue()));
if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
theBool.must(theQb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery());
} else {
throw new InvalidRequestException("Don't know how to handle op=" + nextFilter.getOp() + " on property " + nextFilter.getProperty());
}
}
private void handleLoincCopyrightFilter(QueryBuilder theQb, BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent nextFilter) {
if (nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
String copyrightFilterValue = defaultString(nextFilter.getValue()).toLowerCase();
switch (copyrightFilterValue) {
case "loinc":
ourLog.info(" * Filtering with value=" + nextFilter.getValue() + " on property " + nextFilter.getProperty());
addCopyrightFilterLoinc(theBool);
break;
case "3rdparty":
ourLog.info(" * Filtering with value=" + nextFilter.getValue() + " on property " + nextFilter.getProperty());
addCopyrightFilter3rdParty(theBool);
break;
default:
throw new InvalidRequestException("Don't know how to handle value=" + nextFilter.getValue() + " on property " + nextFilter.getProperty());
}
} else {
throw new InvalidRequestException("Don't know how to handle op=" + nextFilter.getOp() + " on property " + nextFilter.getProperty());
}
}
private void handleRegexFilter(BooleanJunction<?> theBool, ValueSet.ConceptSetFilterComponent nextFilter) {
if (nextFilter.getOp() == ValueSet.FilterOperator.REGEX) {
/*
* We treat the regex filter as a match on the regex
* anywhere in the property string. The spec does not
* say whether or not this is the right behaviour, but
* there are examples that seem to suggest that it is.
*/
String value = nextFilter.getValue();
if (value.endsWith("$")) {
value = value.substring(0, value.length() - 1);
} else if (!value.endsWith(".*")) {
value = value + ".*";
}
if (!value.startsWith("^") && !value.startsWith(".*")) {
value = ".*" + value;
} else if (value.startsWith("^")) {
value = value.substring(1);
}
Term term = new Term(TermConceptPropertyFieldBridge.CONCEPT_FIELD_PROPERTY_PREFIX + nextFilter.getProperty(), value);
RegexpQuery query = new RegexpQuery(term);
theBool.must(query);
} else {
String value = nextFilter.getValue();
Term term = new Term(TermConceptPropertyFieldBridge.CONCEPT_FIELD_PROPERTY_PREFIX + nextFilter.getProperty(), value);
theBool.must(new TermsQuery(term));
}
}
private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, AtomicInteger theCodeCounter) {
ourLog.trace("Hibernate search is not enabled");
if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {

221
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologySvcImplDstu3Test.java
View File

@ -148,6 +148,11 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
code.addPropertyString("HELLO", "12345-2");
cs.getConcepts().add(code);
code = new TermConcept(cs, "47239-9");
code.addPropertyString("SYSTEM", "^Patient");
code.addPropertyString("EXTERNAL_COPYRIGHT_NOTICE", "Copyright © 2006 World Health Organization...");
cs.getConcepts().add(code);
myTermSvc.storeNewCodeSystemVersion(table.getId(), CS_URL, "SYSTEM NAME", "SYSTEM VERSION" , cs);
});
}
@ -273,6 +278,218 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
}
@Test
public void testExpandValueSetPropertyFilterCopyrightWithExclude3rdParty() {
createLoincSystemWithSomeCodes();
List<String> codes;
ValueSet vs;
ValueSet outcome;
ValueSet.ConceptSetComponent exclude;
// Include
vs = new ValueSet();
vs.getCompose()
.addInclude()
.setSystem(CS_URL);
// Exclude
exclude = vs.getCompose().addExclude();
exclude.setSystem(CS_URL);
exclude
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("3rdParty");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("50015-7", "43343-3", "43343-4"));
// Include
vs = new ValueSet();
vs.getCompose()
.addInclude()
.setSystem(CS_URL);
// Exclude
exclude = vs.getCompose().addExclude();
exclude.setSystem(CS_URL);
exclude
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("3rdparty");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("50015-7", "43343-3", "43343-4"));
}
@Test
public void testExpandValueSetPropertyFilterCopyrightWithExcludeLoinc() {
createLoincSystemWithSomeCodes();
List<String> codes;
ValueSet vs;
ValueSet outcome;
ValueSet.ConceptSetComponent exclude;
// Include
vs = new ValueSet();
vs.getCompose()
.addInclude()
.setSystem(CS_URL);
// Exclude
exclude = vs.getCompose().addExclude();
exclude.setSystem(CS_URL);
exclude
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("LOINC");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("47239-9"));
// Include
vs = new ValueSet();
vs.getCompose()
.addInclude()
.setSystem(CS_URL);
// Exclude
exclude = vs.getCompose().addExclude();
exclude.setSystem(CS_URL);
exclude
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("loinc");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("47239-9"));
}
@Test
public void testExpandValueSetPropertyFilterCopyrightWithInclude3rdParty() {
createLoincSystemWithSomeCodes();
List<String> codes;
ValueSet vs;
ValueSet outcome;
ValueSet.ConceptSetComponent include;
// Include
vs = new ValueSet();
include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
include
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("3rdParty");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("47239-9"));
// Include
vs = new ValueSet();
include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
include
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("3rdparty");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("47239-9"));
}
@Test
public void testExpandValueSetPropertyFilterCopyrightWithIncludeLoinc() {
createLoincSystemWithSomeCodes();
List<String> codes;
ValueSet vs;
ValueSet outcome;
ValueSet.ConceptSetComponent include;
// Include
vs = new ValueSet();
include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
include
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("LOINC");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("50015-7", "43343-3", "43343-4"));
// Include
vs = new ValueSet();
include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
include
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("loinc");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("50015-7", "43343-3", "43343-4"));
}
@Test
public void testExpandValueSetPropertyFilterCopyrightWithUnsupportedOp() {
createLoincSystemWithSomeCodes();
List<String> codes;
ValueSet vs;
ValueSet outcome;
ValueSet.ConceptSetComponent include;
// Include
vs = new ValueSet();
include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
include
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.ISA)
.setValue("LOINC");
try {
outcome = myTermSvc.expandValueSet(vs);
} catch (InvalidRequestException e) {
assertEquals(400, e.getStatusCode());
assertEquals("Don't know how to handle op=ISA on property copyright", e.getMessage());
}
}
@Test
public void testExpandValueSetPropertyFilterCopyrightWithUnsupportedValue() {
createLoincSystemWithSomeCodes();
List<String> codes;
ValueSet vs;
ValueSet outcome;
ValueSet.ConceptSetComponent include;
// Include
vs = new ValueSet();
include = vs.getCompose().addInclude();
include.setSystem(CS_URL);
include
.addFilter()
.setProperty("copyright")
.setOp(ValueSet.FilterOperator.EQUAL)
.setValue("bogus");
try {
outcome = myTermSvc.expandValueSet(vs);
} catch (InvalidRequestException e) {
assertEquals(400, e.getStatusCode());
assertEquals("Don't know how to handle value=bogus on property copyright", e.getMessage());
}
}
@Test
public void testExpandValueSetPropertySearchWithRegexExclude() {
createLoincSystemWithSomeCodes();
@ -297,7 +514,7 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
.setValue(".*\\^Donor$");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("43343-3", "43343-4"));
assertThat(codes, containsInAnyOrder("43343-3", "43343-4", "47239-9"));
}
@Test
@ -324,7 +541,7 @@ public class TerminologySvcImplDstu3Test extends BaseJpaDstu3Test {
.setValue("12345-1|12345-2");
outcome = myTermSvc.expandValueSet(vs);
codes = toCodesContains(outcome.getExpansion().getContains());
assertThat(codes, containsInAnyOrder("50015-7"));
assertThat(codes, containsInAnyOrder("50015-7", "47239-9"));
}
@Test