honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix cacheing for large valuesets

This commit is contained in:
dotasek 2021-12-09 15:50:03 -05:00
parent e5976307f4
commit 394452b747
4 changed files with 118 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/TerminologyCache.java
View File

@ -148,7 +148,12 @@ public class TerminologyCache {
public String extracted(JsonParser json, ValueSet vsc) throws IOException {
String s = null;
if (vsc.getExpansion().getContains().size() > 1000 || vsc.getCompose().getIncludeFirstRep().getConcept().size() > 1000) {
s = Integer.toString(vsc.hashCode()); // turn caching off - hack efficiency optimisation
int hashCode = (vsc.getExpansion().getContains().toString() + vsc.getCompose().getIncludeFirstRep().getConcept().toString()).hashCode();
StringBuilder sb = new StringBuilder();
for (ValueSet.ConceptReferenceComponent c : vsc.getCompose().getIncludeFirstRep().getConcept()) {
sb.append(c.getCode());
}
s = Integer.toString((vsc.getExpansion().getContains().toString() + sb.toString()).hashCode()); // turn caching off - hack efficiency optimisation
} else {
s = json.composeString(vsc);
}
@ -380,56 +385,58 @@ public class TerminologyCache {
}
private void load() throws FHIRException {
for (String fn : new File(folder).list()) {
if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
int c = 0;
try {
String title = fn.substring(0, fn.lastIndexOf("."));
NamedCache nc = new NamedCache();
nc.name = title;
caches.put(title, nc);
String src = TextFile.fileToString(Utilities.path(folder, fn));
if (src.startsWith("?"))
src = src.substring(1);
int i = src.indexOf(ENTRY_MARKER);
while (i > -1) {
c++;
String s = src.substring(0, i);
src = src.substring(i+ENTRY_MARKER.length()+1);
i = src.indexOf(ENTRY_MARKER);
if (!Utilities.noString(s)) {
int j = s.indexOf(BREAK);
String q = s.substring(0, j);
String p = s.substring(j+BREAK.length()+1).trim();
CacheEntry ce = new CacheEntry();
ce.persistent = true;
ce.request = q;
boolean e = p.charAt(0) == 'e';
p = p.substring(3);
JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
String error = loadJS(o.get("error"));
if (e) {
if (o.has("valueSet"))
ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
else
ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
} else {
String t = loadJS(o.get("severity"));
IssueSeverity severity = t == null ? null : IssueSeverity.fromCode(t);
String display = loadJS(o.get("display"));
String code = loadJS(o.get("code"));
String system = loadJS(o.get("system"));
String definition = loadJS(o.get("definition"));
t = loadJS(o.get("class"));
TerminologyServiceErrorClass errorClass = t == null ? null : TerminologyServiceErrorClass.valueOf(t) ;
ce.v = new ValidationResult(severity, error, system, new ConceptDefinitionComponent().setDisplay(display).setDefinition(definition).setCode(code)).setErrorClass(errorClass);
synchronized (lock) {
for (String fn : new File(folder).list()) {
if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
int c = 0;
try {
String title = fn.substring(0, fn.lastIndexOf("."));
NamedCache nc = new NamedCache();
nc.name = title;
caches.put(title, nc);
String src = TextFile.fileToString(Utilities.path(folder, fn));
if (src.startsWith("?"))
src = src.substring(1);
int i = src.indexOf(ENTRY_MARKER);
while (i > -1) {
c++;
String s = src.substring(0, i);
src = src.substring(i + ENTRY_MARKER.length() + 1);
i = src.indexOf(ENTRY_MARKER);
if (!Utilities.noString(s)) {
int j = s.indexOf(BREAK);
String q = s.substring(0, j);
String p = s.substring(j + BREAK.length() + 1).trim();
CacheEntry ce = new CacheEntry();
ce.persistent = true;
ce.request = q;
boolean e = p.charAt(0) == 'e';
p = p.substring(3);
JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
String error = loadJS(o.get("error"));
if (e) {
if (o.has("valueSet"))
ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
else
ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
} else {
String t = loadJS(o.get("severity"));
IssueSeverity severity = t == null ? null : IssueSeverity.fromCode(t);
String display = loadJS(o.get("display"));
String code = loadJS(o.get("code"));
String system = loadJS(o.get("system"));
String definition = loadJS(o.get("definition"));
t = loadJS(o.get("class"));
TerminologyServiceErrorClass errorClass = t == null ? null : TerminologyServiceErrorClass.valueOf(t);
ce.v = new ValidationResult(severity, error, system, new ConceptDefinitionComponent().setDisplay(display).setDefinition(definition).setCode(code)).setErrorClass(errorClass);
}
nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
nc.list.add(ce);
}
nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
nc.list.add(ce);
}
} catch (Exception e) {
throw new FHIRException("Error loading " + fn + ": " + e.getMessage() + " entry " + c, e);
}
} catch (Exception e) {
throw new FHIRException("Error loading "+fn+": "+e.getMessage()+" entry "+c, e);
}
}
}

1
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/terminologies/ValueSetCheckerSimple.java
View File

@ -148,6 +148,7 @@ public class ValueSetCheckerSimple extends ValueSetWorker implements ValueSetChe
if (cs == null) {
cs = findSpecialCodeSystem(system);
}
System.out.print("");
if (cs == null) {
warningMessage = "Unable to resolve system "+system;
if (!inExpansion) {

5
org.hl7.fhir.validation/src/test/java/org/hl7/fhir/TestingUtilities.java
View File

@ -5,15 +5,16 @@ import org.hl7.fhir.validation.ValidationEngine;
public class TestingUtilities {
public static final ValidationEngine getValidationEngine(java.lang.String src, java.lang.String txsrvr, java.lang.String txLog, FhirPublication version, boolean canRunWithoutTerminologyServer, java.lang.String vString, java.lang.String userAgent) throws Exception {
txLog = TestConstants.TX_CACHE + "/tx.log.html";
final ValidationEngine validationEngine = new ValidationEngine(src, txsrvr, txLog, version, canRunWithoutTerminologyServer, vString, userAgent);
validationEngine.getContext().initTS(TestConstants.TX_CACHE);
validationEngine.getContext().initTS(TestConstants.TX_CACHE + "/" + version.toString());
validationEngine.getContext().setUserAgent("fhir/test-cases");
return validationEngine;
}
public static ValidationEngine getValidationEngine(java.lang.String src, java.lang.String txsrvr, java.lang.String txLog, FhirPublication version, java.lang.String vString, java.lang.String userAgent) throws Exception {
final ValidationEngine validationEngine = new ValidationEngine(src, txsrvr, txLog, version, vString, userAgent);
validationEngine.getContext().initTS(TestConstants.TX_CACHE);
validationEngine.getContext().initTS(TestConstants.TX_CACHE + "/" + version.toString());
validationEngine.getContext().setUserAgent("fhir/test-cases");
return validationEngine;
}

1
org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ValidationTests.java
View File

@ -101,6 +101,7 @@ public class ValidationTests implements IEvaluationContext, IValidatorResourceFe
List<Object[]> objects = new ArrayList<Object[]>(examples.size());
for (String id : names) {
//if ("dr-eh".equals(id))
objects.add(new Object[]{id, examples.get(id)});
}
return objects;