honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed the pre-expansion with filter issue (#2257) 

* Fixed the pre-expansion with filter issue

* Changed minGramSize to 3 instead of 2
This commit is contained in:
Frank Tao 2020-12-17 23:31:33 -05:00 committed by GitHub
parent e79114e2ec
commit 89cf6e6293
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 96 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/LuceneSearchMappingFactory.java
View File

@ -52,7 +52,7 @@ public class LuceneSearchMappingFactory {
.filter(LowerCaseFilterFactory.class)
.filter(StopFilterFactory.class)
.filter(EdgeNGramFilterFactory.class)
.param("minGramSize", "2")
.param("minGramSize", "3")
.param("maxGramSize", "20")
.analyzerDef("autocompletePhoneticAnalyzer", StandardTokenizerFactory.class)
.filter(StandardFilterFactory.class)

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchMappingProvider.java
View File

@ -41,7 +41,7 @@ public class ElasticsearchMappingProvider implements ElasticsearchAnalysisDefini
.withTokenFilters("lowercase", "stop", "wordedgengram_3_50");
builder.tokenFilter("wordedgengram_3_50")
.type("edgeNGram")
.param("min_gram", "2")
.param("min_gram", "3")
.param("max_gram", "20");
builder.analyzer("autocompletePhoneticAnalyzer")

36
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java
View File

@ -172,6 +172,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@ -530,9 +531,10 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
Collection<TermValueSetConceptView> conceptViews;
boolean wasFilteredResult = false;
String filterDisplayValue = null;
if (!theFilter.getFilters().isEmpty() && JpaConstants.VALUESET_FILTER_DISPLAY.equals(theFilter.getFilters().get(0).getProperty()) && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) {
String displayValue = theFilter.getFilters().get(0).getValue().replace("%", "[%]") + "%";
displayValue = lowerCase(displayValue);
filterDisplayValue = lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]"));
String displayValue = "%" + lowerCase(filterDisplayValue) + "%";
conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
wasFilteredResult = true;
} else {
@ -553,11 +555,16 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
for (TermValueSetConceptView conceptView : conceptViews) {
Long conceptPid = conceptView.getConceptPid();
if (!pidToConcept.containsKey(conceptPid)) {
String system = conceptView.getConceptSystemUrl();
String code = conceptView.getConceptCode();
String display = conceptView.getConceptDisplay();
//-- this is quick solution, may need to revisit
if (!applyFilter(display, filterDisplayValue))
continue;
Long conceptPid = conceptView.getConceptPid();
if (!pidToConcept.containsKey(conceptPid)) {
FhirVersionIndependentConcept concept = new FhirVersionIndependentConcept(system, code, display);
pidToConcept.put(conceptPid, concept);
}
@ -625,6 +632,27 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
}
}
public boolean applyFilter(final String theInput, final String thePrefixToken) {
//-- safety check only, no need to apply filter
if (theInput == null || thePrefixToken == null)
return true;
// -- sentence case
if (org.apache.commons.lang3.StringUtils.startsWithIgnoreCase(theInput, thePrefixToken))
return true;
//-- token case
// return true only e.g. the input is 'Body height', thePrefixToken is "he", or 'bo'
StringTokenizer tok = new StringTokenizer(theInput);
while (tok.hasMoreTokens()) {
if (org.apache.commons.lang3.StringUtils.startsWithIgnoreCase(tok.nextToken(), thePrefixToken))
return true;
}
return false;
}
@Override
@Transactional(propagation = Propagation.REQUIRED)
public void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) {

56
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4ValueSetNoVerCSNoVerTest.java
View File

@ -312,17 +312,69 @@ public class ResourceProviderR4ValueSetNoVerCSNoVerTest extends BaseResourceProv
.operation()
.onInstance(myExtensionalVsId)
.named("expand")
.withParameter(Parameters.class, "filter", new StringType("systolic"))
.withParameter(Parameters.class, "filter", new StringType("blood"))
.execute();
ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
ourLog.info(resp);
assertThat(resp, containsString("<display value=\"Systolic blood pressure at First encounter\"/>"));
assertThat(resp, not(containsString("\"Foo Code\"")));
}
@Test
public void testExpandByIdWithFilterWithPreExpansionWithPrefixValue() throws Exception {
myDaoConfig.setPreExpandValueSets(true);
loadAndPersistCodeSystemAndValueSet();
await().until(() -> clearDeferredStorageQueue());
myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(PageRequest.of(0, 10), TermValueSetPreExpansionStatusEnum.EXPANDED);
assertEquals(1, page.getContent().size());
Parameters respParam = myClient
.operation()
.onInstance(myExtensionalVsId)
.named("expand")
.withParameter(Parameters.class, "filter", new StringType("blo"))
.execute();
ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
ourLog.info(resp);
assertThat(resp, containsString("<display value=\"Systolic blood pressure at First encounter\"/>"));
assertThat(resp, not(containsString("\"Foo Code\"")));
}
@Test
public void testExpandByIdWithFilterWithPreExpansionWithoutPrefixValue() throws Exception {
myDaoConfig.setPreExpandValueSets(true);
loadAndPersistCodeSystemAndValueSet();
await().until(() -> clearDeferredStorageQueue());
myTermSvc.preExpandDeferredValueSetsToTerminologyTables();
Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(PageRequest.of(0, 10), TermValueSetPreExpansionStatusEnum.EXPANDED);
assertEquals(1, page.getContent().size());
Parameters respParam = myClient
.operation()
.onInstance(myExtensionalVsId)
.named("expand")
.withParameter(Parameters.class, "filter", new StringType("lood"))
.execute();
ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource();
String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded);
ourLog.info(resp);
assertThat(resp, not(containsString("<display value=\"Systolic blood pressure at First encounter\"/>")));
assertThat(resp, not(containsString("\"Foo Code\"")));
}
@Test
public void testExpandByUrl() throws Exception {
loadAndPersistCodeSystemAndValueSet();

14
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4Test.java
View File

@ -189,7 +189,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString(" like 'display value 9%'"));
assertThat(lastSelectQuery, containsString(" like '%display value 9%'"));
}
@ -220,7 +220,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
ourLog.info("SQL: {}", lastSelectQuery);
assertThat(lastSelectQuery, containsString(" like 'display value 100%'"));
assertThat(lastSelectQuery, containsString(" like '%display value 100%'"));
}
// ValueSet by ID
@ -234,7 +234,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
ourLog.info("SQL: {}", lastSelectQuery);
assertThat(lastSelectQuery, containsString(" like 'display value 100%'"));
assertThat(lastSelectQuery, containsString(" like '%display value 100%'"));
}
}
@ -269,7 +269,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString(" like 'display value 9%'"));
assertThat(lastSelectQuery, containsString(" like '%display value 9%'"));
}
@ -297,7 +297,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString("like 'display value 99%'"));
assertThat(lastSelectQuery, containsString("like '%display value 99%'"));
}
@ -335,7 +335,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString(" like 'display value 90%'"));
assertThat(lastSelectQuery, containsString(" like '%display value 90%'"));
}
@ -434,7 +434,7 @@ public class ValueSetExpansionR4Test extends BaseTermR4Test {
// Make sure we used the pre-expanded version
List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueries();
String lastSelectQuery = selectQueries.get(selectQueries.size() - 1).getSql(true, true).toLowerCase();
assertThat(lastSelectQuery, containsString(" like 'display value 9%'"));
assertThat(lastSelectQuery, containsString(" like '%display value 9%'"));
}
@Nonnull