honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix cacheing for large valuesets

This commit is contained in:
dotasek 2021-12-09 15:50:03 -05:00
parent e5976307f4
commit 394452b747
4 changed files with 118 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/TerminologyCache.java
View File

@ -148,7 +148,12 @@ public class TerminologyCache {
public String extracted(JsonParser json, ValueSet vsc) throws IOException { public String extracted(JsonParser json, ValueSet vsc) throws IOException {
String s = null; String s = null;
if (vsc.getExpansion().getContains().size() > 1000 || vsc.getCompose().getIncludeFirstRep().getConcept().size() > 1000) { if (vsc.getExpansion().getContains().size() > 1000 || vsc.getCompose().getIncludeFirstRep().getConcept().size() > 1000) {
s = Integer.toString(vsc.hashCode()); // turn caching off - hack efficiency optimisation int hashCode = (vsc.getExpansion().getContains().toString() + vsc.getCompose().getIncludeFirstRep().getConcept().toString()).hashCode();
StringBuilder sb = new StringBuilder();
for (ValueSet.ConceptReferenceComponent c : vsc.getCompose().getIncludeFirstRep().getConcept()) {
sb.append(c.getCode());
}
s = Integer.toString((vsc.getExpansion().getContains().toString() + sb.toString()).hashCode()); // turn caching off - hack efficiency optimisation
} else { } else {
s = json.composeString(vsc); s = json.composeString(vsc);
} }
@ -380,56 +385,58 @@ public class TerminologyCache {
} }
private void load() throws FHIRException { private void load() throws FHIRException {
for (String fn : new File(folder).list()) { synchronized (lock) {
if (fn.endsWith(".cache") && !fn.equals("validation.cache")) { for (String fn : new File(folder).list()) {
int c = 0; if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
try { int c = 0;
String title = fn.substring(0, fn.lastIndexOf(".")); try {
NamedCache nc = new NamedCache(); String title = fn.substring(0, fn.lastIndexOf("."));
nc.name = title; NamedCache nc = new NamedCache();
caches.put(title, nc); nc.name = title;
String src = TextFile.fileToString(Utilities.path(folder, fn)); caches.put(title, nc);
if (src.startsWith("?")) String src = TextFile.fileToString(Utilities.path(folder, fn));
src = src.substring(1); if (src.startsWith("?"))
int i = src.indexOf(ENTRY_MARKER); src = src.substring(1);
while (i > -1) { int i = src.indexOf(ENTRY_MARKER);
c++; while (i > -1) {
String s = src.substring(0, i); c++;
src = src.substring(i+ENTRY_MARKER.length()+1); String s = src.substring(0, i);
i = src.indexOf(ENTRY_MARKER); src = src.substring(i + ENTRY_MARKER.length() + 1);
if (!Utilities.noString(s)) { i = src.indexOf(ENTRY_MARKER);
int j = s.indexOf(BREAK); if (!Utilities.noString(s)) {
String q = s.substring(0, j); int j = s.indexOf(BREAK);
String p = s.substring(j+BREAK.length()+1).trim(); String q = s.substring(0, j);
CacheEntry ce = new CacheEntry(); String p = s.substring(j + BREAK.length() + 1).trim();
ce.persistent = true; CacheEntry ce = new CacheEntry();
ce.request = q; ce.persistent = true;
boolean e = p.charAt(0) == 'e'; ce.request = q;
p = p.substring(3); boolean e = p.charAt(0) == 'e';
JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p); p = p.substring(3);
String error = loadJS(o.get("error")); JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
if (e) { String error = loadJS(o.get("error"));
if (o.has("valueSet")) if (e) {
ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN); if (o.has("valueSet"))
else ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN); else
} else { ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
String t = loadJS(o.get("severity")); } else {
IssueSeverity severity = t == null ? null : IssueSeverity.fromCode(t); String t = loadJS(o.get("severity"));
String display = loadJS(o.get("display")); IssueSeverity severity = t == null ? null : IssueSeverity.fromCode(t);
String code = loadJS(o.get("code")); String display = loadJS(o.get("display"));
String system = loadJS(o.get("system")); String code = loadJS(o.get("code"));
String definition = loadJS(o.get("definition")); String system = loadJS(o.get("system"));
t = loadJS(o.get("class")); String definition = loadJS(o.get("definition"));
TerminologyServiceErrorClass errorClass = t == null ? null : TerminologyServiceErrorClass.valueOf(t) ; t = loadJS(o.get("class"));
ce.v = new ValidationResult(severity, error, system, new ConceptDefinitionComponent().setDisplay(display).setDefinition(definition).setCode(code)).setErrorClass(errorClass); TerminologyServiceErrorClass errorClass = t == null ? null : TerminologyServiceErrorClass.valueOf(t);
ce.v = new ValidationResult(severity, error, system, new ConceptDefinitionComponent().setDisplay(display).setDefinition(definition).setCode(code)).setErrorClass(errorClass);
}
nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
nc.list.add(ce);
} }
nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
nc.list.add(ce);
} }
} catch (Exception e) {
throw new FHIRException("Error loading " + fn + ": " + e.getMessage() + " entry " + c, e);
} }
} catch (Exception e) {
throw new FHIRException("Error loading "+fn+": "+e.getMessage()+" entry "+c, e);
} }
} }
} }

1
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/terminologies/ValueSetCheckerSimple.java
View File

@ -148,6 +148,7 @@ public class ValueSetCheckerSimple extends ValueSetWorker implements ValueSetChe
if (cs == null) { if (cs == null) {
cs = findSpecialCodeSystem(system); cs = findSpecialCodeSystem(system);
} }
System.out.print("");
if (cs == null) { if (cs == null) {
warningMessage = "Unable to resolve system "+system; warningMessage = "Unable to resolve system "+system;
if (!inExpansion) { if (!inExpansion) {

5
org.hl7.fhir.validation/src/test/java/org/hl7/fhir/TestingUtilities.java
View File

@ -5,15 +5,16 @@ import org.hl7.fhir.validation.ValidationEngine;
public class TestingUtilities { public class TestingUtilities {
public static final ValidationEngine getValidationEngine(java.lang.String src, java.lang.String txsrvr, java.lang.String txLog, FhirPublication version, boolean canRunWithoutTerminologyServer, java.lang.String vString, java.lang.String userAgent) throws Exception { public static final ValidationEngine getValidationEngine(java.lang.String src, java.lang.String txsrvr, java.lang.String txLog, FhirPublication version, boolean canRunWithoutTerminologyServer, java.lang.String vString, java.lang.String userAgent) throws Exception {
txLog = TestConstants.TX_CACHE + "/tx.log.html";
final ValidationEngine validationEngine = new ValidationEngine(src, txsrvr, txLog, version, canRunWithoutTerminologyServer, vString, userAgent); final ValidationEngine validationEngine = new ValidationEngine(src, txsrvr, txLog, version, canRunWithoutTerminologyServer, vString, userAgent);
validationEngine.getContext().initTS(TestConstants.TX_CACHE); validationEngine.getContext().initTS(TestConstants.TX_CACHE + "/" + version.toString());
validationEngine.getContext().setUserAgent("fhir/test-cases"); validationEngine.getContext().setUserAgent("fhir/test-cases");
return validationEngine; return validationEngine;
} }
public static ValidationEngine getValidationEngine(java.lang.String src, java.lang.String txsrvr, java.lang.String txLog, FhirPublication version, java.lang.String vString, java.lang.String userAgent) throws Exception { public static ValidationEngine getValidationEngine(java.lang.String src, java.lang.String txsrvr, java.lang.String txLog, FhirPublication version, java.lang.String vString, java.lang.String userAgent) throws Exception {
final ValidationEngine validationEngine = new ValidationEngine(src, txsrvr, txLog, version, vString, userAgent); final ValidationEngine validationEngine = new ValidationEngine(src, txsrvr, txLog, version, vString, userAgent);
validationEngine.getContext().initTS(TestConstants.TX_CACHE); validationEngine.getContext().initTS(TestConstants.TX_CACHE + "/" + version.toString());
validationEngine.getContext().setUserAgent("fhir/test-cases"); validationEngine.getContext().setUserAgent("fhir/test-cases");
return validationEngine; return validationEngine;
} }

1
org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ValidationTests.java
View File

@ -101,6 +101,7 @@ public class ValidationTests implements IEvaluationContext, IValidatorResourceFe
List<Object[]> objects = new ArrayList<Object[]>(examples.size()); List<Object[]> objects = new ArrayList<Object[]>(examples.size());
for (String id : names) { for (String id : names) {
//if ("dr-eh".equals(id))
objects.add(new Object[]{id, examples.get(id)}); objects.add(new Object[]{id, examples.get(id)});
} }
return objects; return objects;