work on loading speed for validator

This commit is contained in:
Grahame Grieve 2022-11-21 18:46:47 -03:00
parent 68360cb110
commit 8136b00c86
12 changed files with 101 additions and 57 deletions

View File

@ -105,7 +105,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
private IContextResourceLoader loader;
public PackageResourceLoader(PackageResourceInformation pri, IContextResourceLoader loader) {
super(pri.getType(), pri.getId(), pri.getUrl(),pri.getVersion());
super(pri.getResourceType(), pri.getId(), pri.getUrl(),pri.getVersion());
this.filename = pri.getFilename();
this.loader = loader;
}

View File

@ -1,12 +1,16 @@
package org.hl7.fhir.r5.conformance;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.r5.conformance.R5ExtensionsLoader.Loadable;
import org.hl7.fhir.r5.context.ContextUtilities;
import org.hl7.fhir.r5.context.IWorkerContext;
import org.hl7.fhir.r5.context.IWorkerContext.PackageVersion;
@ -28,15 +32,40 @@ import org.hl7.fhir.utilities.npm.NpmPackage;
import org.hl7.fhir.utilities.npm.NpmPackage.PackageResourceInformation;
public class R5ExtensionsLoader {
public static class CanonicalResourceSortByUrl<T extends CanonicalResource> implements Comparator<Loadable<T>> {
@Override
public int compare(Loadable<T> arg0, Loadable<T> arg1) {
return arg0.info.getUrl().compareTo(arg1.info.getUrl());
}
}
public class Loadable<T extends CanonicalResource> {
public Loadable(PackageResourceInformation info) {
this.info = info;
}
private T resource;
private PackageResourceInformation info;
public T getResource() throws FHIRFormatError, FileNotFoundException, IOException {
if (resource == null) {
CanonicalResource r = (CanonicalResource) json.parse(pck.load(info));
r.setUserData("path", Utilities.pathURL(pck.getWebLocation(), r.fhirType().toLowerCase()+ "-"+r.getId().toLowerCase()+".html"));
resource = (T) r;
}
return resource;
}
}
private BasePackageCacheManager pcm;
private int count;
private byte[] map;
private NpmPackage pck;
private Map<String, ValueSet> valueSets;
private Map<String, CodeSystem> codeSystems;
private List<StructureDefinition> structures;
private Map<String, Loadable<ValueSet>> valueSets;
private Map<String, Loadable<CodeSystem>> codeSystems;
private List<Loadable<StructureDefinition>> structures;
private IWorkerContext context;
private PackageVersion pd;
private JsonParser json;
public R5ExtensionsLoader(BasePackageCacheManager pcm, IWorkerContext context) {
super();
@ -51,45 +80,43 @@ public class R5ExtensionsLoader {
public void load() throws FHIRException, IOException {
pck = pcm.loadPackage("hl7.fhir.r5.core", "current");
pd = new PackageVersion(pck.name(), pck.version(), pck.dateAsDate());
map = pck.hasFile("other", "spec.internals") ? TextFile.streamToBytes(pck.load("other", "spec.internals")) : null;
String[] types = new String[] { "StructureDefinition", "ValueSet", "CodeSystem" };
JsonParser json = new JsonParser();
json = new JsonParser();
for (PackageResourceInformation pri : pck.listIndexedResources(types)) {
CanonicalResource r = (CanonicalResource) json.parse(pck.load(pri));
r.setUserData("path", Utilities.pathURL(pck.getWebLocation(), r.fhirType().toLowerCase()+ "-"+r.getId().toLowerCase()+".html"));
if (r instanceof CodeSystem) {
codeSystems.put(r.getUrl(), (CodeSystem) r);
codeSystems.put(r.getUrl()+"|"+r.getVersion(), (CodeSystem) r);
} else if (r instanceof ValueSet) {
valueSets.put(r.getUrl(), (ValueSet) r);
valueSets.put(r.getUrl()+"|"+r.getVersion(), (ValueSet) r);
} else if (r instanceof StructureDefinition) {
structures.add((StructureDefinition) r);
if (pri.getResourceType().equals("CodeSystem")) {
codeSystems.put(pri.getUrl(), new Loadable<CodeSystem>(pri));
codeSystems.put(pri.getUrl()+"|"+pri.getVersion(), new Loadable<CodeSystem>(pri));
} else if (pri.getResourceType().equals("ValueSet")) {
valueSets.put(pri.getUrl(), new Loadable<ValueSet>(pri));
valueSets.put(pri.getUrl()+"|"+pri.getVersion(), new Loadable<ValueSet>(pri));
} else if (pri.getResourceType().equals("StructureDefinition") && pri.getStatedType().equals("Extension")) {
structures.add(new Loadable<StructureDefinition>(pri));
}
}
structures.sort(new ResourceSorters.CanonicalResourceSortByUrl());
}
public void loadR5Extensions() throws FHIRException, IOException {
count = 0;
List<String> typeNames = new ContextUtilities(context).getTypeNames();
for (StructureDefinition sd : structures) {
if (sd.getType().equals("Extension") && sd.getDerivation() == TypeDerivationRule.CONSTRAINT &&
!context.hasResource(StructureDefinition.class, sd.getUrl())) {
if (survivesStrippingTypes(sd, context, typeNames)) {
count++;
sd.setUserData("path", Utilities.pathURL(pck.getWebLocation(), "extension-"+sd.getId().toLowerCase()+".html"));
registerTerminologies(sd);
context.cacheResourceFromPackage(sd, pd);
for (Loadable<StructureDefinition> lsd : structures) {
if (!context.hasResource(StructureDefinition.class, lsd.info.getUrl())) {
StructureDefinition sd = lsd.getResource();
if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT) {
if (survivesStrippingTypes(sd, context, typeNames)) {
count++;
sd.setUserData("path", Utilities.pathURL(pck.getWebLocation(), "extension-"+sd.getId().toLowerCase()+".html"));
registerTerminologies(sd);
context.cacheResourceFromPackage(sd, pd);
}
}
}
}
}
public void loadR5SpecialTypes(List<String> types) throws FHIRException, IOException {
for (StructureDefinition sd : structures) {
for (Loadable<StructureDefinition> lsd : structures) {
StructureDefinition sd = lsd.getResource();
if (Utilities.existsInList(sd.getType(), types)) {
count++;
sd.setUserData("path", Utilities.pathURL(pck.getWebLocation(), sd.getId().toLowerCase()+".html"));
@ -99,7 +126,7 @@ public class R5ExtensionsLoader {
}
}
private void registerTerminologies(StructureDefinition sd) {
private void registerTerminologies(StructureDefinition sd) throws FHIRFormatError, FileNotFoundException, IOException {
for (ElementDefinition ed : sd.getSnapshot().getElement()) {
if (ed.hasBinding() && ed.getBinding().hasValueSet()) {
String vsu = ed.getBinding().getValueSet();
@ -113,9 +140,9 @@ public class R5ExtensionsLoader {
}
}
private void loadValueSet(String url, IWorkerContext context, Map<String, ValueSet> valueSets, Map<String, CodeSystem> codeSystems, PackageVersion pd) {
private void loadValueSet(String url, IWorkerContext context, Map<String, Loadable<ValueSet>> valueSets, Map<String, Loadable<CodeSystem>> codeSystems, PackageVersion pd) throws FHIRFormatError, FileNotFoundException, IOException {
if (valueSets.containsKey(url)) {
ValueSet vs = valueSets.get(url);
ValueSet vs = valueSets.get(url).getResource();
context.cacheResourceFromPackage(vs, pd);
for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
for (CanonicalType t : inc.getValueSet()) {
@ -123,11 +150,11 @@ public class R5ExtensionsLoader {
}
if (inc.hasSystem() && !inc.hasVersion()) {
if (codeSystems.containsKey(inc.getSystem())) {
CodeSystem cs = codeSystems.get(inc.getSystem());
CodeSystem cs = codeSystems.get(inc.getSystem()).getResource();
inc.setVersion(cs.getVersion());
context.cacheResourceFromPackage(cs, pd);
} else if (!context.hasResource(CodeSystem.class, inc.getSystem()) && codeSystems.containsKey(inc.getSystem())) {
context.cacheResourceFromPackage(codeSystems.get(inc.getSystem()), pd);
context.cacheResourceFromPackage(codeSystems.get(inc.getSystem()).getResource(), pd);
}
}
}
@ -174,8 +201,8 @@ public class R5ExtensionsLoader {
return count;
}
public byte[] getMap() {
return map;
public byte[] getMap() throws IOException {
return pck.hasFile("other", "spec.internals") ? TextFile.streamToBytes(pck.load("other", "spec.internals")) : null;
}
public NpmPackage getPck() {

View File

@ -2221,6 +2221,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
if (!hasResource(StructureDefinition.class, "http://hl7.org/fhir/StructureDefinition/Base")) {
cacheResource(ProfileUtilities.makeBaseDefinition(version));
}
System.out.print(".");
for (StructureDefinition sd : listStructures()) {
try {
if (sd.getSnapshot().isEmpty()) {
@ -2231,6 +2232,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
System.out.println("Unable to generate snapshot for "+tail(sd.getUrl()) +" from "+tail(sd.getBaseDefinition())+" because "+e.getMessage());
}
}
System.out.print(":");
codeSystems.setVersion(version);
valueSets.setVersion(version);
maps.setVersion(version);

View File

@ -102,7 +102,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
private final IContextResourceLoader loader;
public PackageResourceLoader(PackageResourceInformation pri, IContextResourceLoader loader) {
super(pri.getType(), pri.getId(), pri.getUrl(),pri.getVersion());
super(pri.getResourceType(), pri.getId(), pri.getUrl(),pri.getVersion());
this.filename = pri.getFilename();
this.loader = loader;
}

View File

@ -98,7 +98,7 @@ import org.slf4j.LoggerFactory;
public abstract class JsonParserBase extends ParserBase implements IParser {
static {
LoggerFactory.getLogger("org.hl7.fhir.r5.formats.JsonParserBase").debug("JSON Parser is being loaded");
// LoggerFactory.getLogger("org.hl7.fhir.r5.formats.JsonParserBase").debug("JSON Parser is being loaded");
ClassesLoadedFlags.ourJsonParserBaseLoaded = true;
}

View File

@ -748,6 +748,8 @@ public class I18nConstants {
public static final String SD_PATH_TYPE_MISMATCH = "SD_PATH_TYPE_MISMATCH";
public static final String XHTML_XHTML_Image_Reference_Illegal = "XHTML_XHTML_Image_Reference_Illegal";
public static final String XHTML_XHTML_Entity_Illegal = "XHTML_XHTML_Entity_Illegal";
public static final String UNABLE_TO_RESOLVE_CONTENT_REFERENCE = "UNABLE_TO_RESOLVE_CONTENT_REFERENCE";
public static final String UNABLE_TO_RESOLVE_CONTENT_REFERENCE_IN_THIS_CONTEXT = "UNABLE_TO_RESOLVE_CONTENT_REFERENCE_IN_THIS_CONTEXT";
}

View File

@ -102,26 +102,31 @@ public class NpmPackage {
public class PackageResourceInformation {
private String id;
private String type;
private String resourceType;
private String url;
private String version;
private String filename;
private String supplements;
private String stype;
public PackageResourceInformation(String root, JsonObject fi) throws IOException {
super();
id = JsonUtilities.str(fi, "id");
type = JsonUtilities.str(fi, "resourceType");
resourceType = JsonUtilities.str(fi, "resourceType");
url = JsonUtilities.str(fi, "url");
version = JsonUtilities.str(fi, "version");
filename = Utilities.path(root, JsonUtilities.str(fi, "filename"));
supplements = JsonUtilities.str(fi, "supplements");
stype = JsonUtilities.str(fi, "type");
}
public String getId() {
return id;
}
public String getType() {
return type;
public String getResourceType() {
return resourceType;
}
public String getStatedType() {
return stype;
}
public String getUrl() {
return url;

View File

@ -2,7 +2,7 @@
Bad_file_path_error = \n********************\n* The file name you passed in, ''{0}'', doesn''t exist on the local filesystem.\n* Please verify that this is valid file location.\n********************\n\n
Bundle_BUNDLE_Entry_Canonical = The canonical URL ({0}) cannot match the fullUrl ({1}) unless on the canonical server itself
Bundle_BUNDLE_Entry_Document = The first entry in a document must be a composition
Bundle_BUNDLE_Entry_IdUrlMismatch = Resource ID does not match the ID in the entry full URL (''{0}'' vs ''{1}'')
Bundle_BUNDLE_Entry_IdUrlMismatch = Resource ID does not match the ID in the entry fullUrl (''{0}'' vs ''{1}'')
Bundle_BUNDLE_Entry_MismatchIdUrl = The canonical URL ({0}) cannot match the fullUrl ({1}) unless the resource id ({2}) also matches
Bundle_BUNDLE_Entry_NoFirst = Documents or Messages must contain at least one entry
Bundle_BUNDLE_Entry_NoFirstResource = No resource on first entry
@ -133,8 +133,8 @@ Resource_RES_ID_Missing = Resource requires an id, but none is present
Resource_RES_ID_Prohibited = Resource has an id, but none is allowed
Terminology_PassThrough_TX_Message = {0} for ''{1}#{2}''
Terminology_TX_Binding_CantCheck = Binding by URI reference cannot be checked
Terminology_TX_Binding_Missing = Binding for {0} missing (cc)
Terminology_TX_Binding_Missing2 = Binding for {0} missing
Terminology_TX_Binding_Missing = Binding for CodeableConcept {0} missing
Terminology_TX_Binding_Missing2 = Binding for Coding {0} missing
Terminology_TX_Binding_NoServer = The value provided could not be validated in the absence of a terminology server
Terminology_TX_Binding_NoSource = Binding for path {0} has no source, so can''t be checked
Terminology_TX_Binding_NoSource2 = Binding has no source, so can''t be checked
@ -637,7 +637,7 @@ BUNDLE_BUNDLE_ENTRY_MULTIPLE_PROFILES_one =
BUNDLE_BUNDLE_ENTRY_MULTIPLE_PROFILES_other = {0} profiles found for {1} resource. More than one is not supported at this time. (Type {2}: {3})
RENDER_BUNDLE_HEADER_ROOT = Bundle {0} of type {1}
RENDER_BUNDLE_HEADER_ENTRY = Entry {0}
RENDER_BUNDLE_HEADER_ENTRY_URL = Entry {0} - Full URL = {1}
RENDER_BUNDLE_HEADER_ENTRY_URL = Entry {0} - fullUrl = {1}
RENDER_BUNDLE_RESOURCE = Resource {0}:
RENDER_BUNDLE_SEARCH = Search:
RENDER_BUNDLE_SEARCH_MODE = mode = {0}
@ -798,3 +798,5 @@ SD_CONSTRAINED_TYPE_NO_MATCH = The type {0} must be the same as the type in the
SD_SPECIALIZED_TYPE_MATCHES = The type {0} must not be the same as the type in the base structure {1} that is being specialised
SD_CONSTRAINED_KIND_NO_MATCH = The kind {0} must be the same as the kind {1} in the base structure {3} (base type = {2})
SD_PATH_TYPE_MISMATCH = The path {1} must start with the type of the structure {0}
UNABLE_TO_RESOLVE_CONTENT_REFERENCE = Unable to resolve the content reference {0} on element {1} (path = {2})
UNABLE_TO_RESOLVE_CONTENT_REFERENCE_IN_THIS_CONTEXT = Unable to resolve the content reference {0} on element {1} (path = {2}) in this context

View File

@ -313,6 +313,9 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
@With
private final IWorkerContext.ILoggingService loggingService;
@With
private boolean THO = true;
public ValidationEngineBuilder() {
terminologyCachePath = null;
@ -326,7 +329,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
loggingService = new SystemOutLoggingService();
}
public ValidationEngineBuilder(String terminologyCachePath, String userAgent, String version, String txServer, String txLog, FhirPublication txVersion, TimeTracker timeTracker, boolean canRunWithoutTerminologyServer, IWorkerContext.ILoggingService loggingService) {
public ValidationEngineBuilder(String terminologyCachePath, String userAgent, String version, String txServer, String txLog, FhirPublication txVersion, TimeTracker timeTracker, boolean canRunWithoutTerminologyServer, IWorkerContext.ILoggingService loggingService, boolean THO) {
this.terminologyCachePath = terminologyCachePath;
this.userAgent = userAgent;
this.version = version;
@ -336,10 +339,11 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
this.timeTracker = timeTracker;
this.canRunWithoutTerminologyServer = canRunWithoutTerminologyServer;
this.loggingService = loggingService;
this.THO = THO;
}
public ValidationEngineBuilder withTxServer(String txServer, String txLog, FhirPublication txVersion) {
return new ValidationEngineBuilder(terminologyCachePath, userAgent, version, txServer, txLog, txVersion,timeTracker, canRunWithoutTerminologyServer, loggingService);
return new ValidationEngineBuilder(terminologyCachePath, userAgent, version, txServer, txLog, txVersion, timeTracker, canRunWithoutTerminologyServer, loggingService, THO);
}
public ValidationEngine fromNothing() throws IOException {
@ -364,7 +368,9 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
}
engine.setVersion(version);
engine.setIgLoader(new IgLoader(engine.getPcm(), engine.getContext(), engine.getVersion(), engine.isDebug()));
loadTx(engine);
if (THO) {
loadTx(engine);
}
return engine;
}

View File

@ -52,22 +52,22 @@ public class ValidatorUtils {
return null;
}
if (VersionUtilities.isR2Ver(version)) {
return new R2ToR5Loader(new String[]{"Conformance", "StructureDefinition", "ValueSet", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5());
return new R2ToR5Loader(new String[]{"Conformance", "StructureDefinition", "ValueSet", /* "SearchParameter", "OperationDefinition",*/ "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5());
}
if (VersionUtilities.isR2BVer(version)) {
return new R2016MayToR5Loader(new String[]{"Conformance", "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5()); // special case
return new R2016MayToR5Loader(new String[]{"Conformance", "StructureDefinition", "ValueSet", "CodeSystem", /*"SearchParameter", "OperationDefinition",*/ "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5()); // special case
}
if (VersionUtilities.isR3Ver(version)) {
return new R3ToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5());
return new R3ToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", /*"SearchParameter", "OperationDefinition",*/ "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5());
}
if (VersionUtilities.isR4Ver(version)) {
return new R4ToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5(), version);
return new R4ToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", /*"SearchParameter", "OperationDefinition",*/ "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5(), version);
}
if (VersionUtilities.isR4BVer(version)) {
return new R4BToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5(), version);
return new R4BToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", /*"SearchParameter", "OperationDefinition",*/ "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5(), version);
}
if (VersionUtilities.isR5Ver(version)) {
return new R5ToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5());
return new R5ToR5Loader(new String[]{"CapabilityStatement", "StructureDefinition", "ValueSet", "CodeSystem", /*"SearchParameter", "OperationDefinition",*/ "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem"}, new NullLoaderKnowledgeProviderR5());
}
return null;
}

View File

@ -364,13 +364,13 @@ public class ValidationService {
System.out.println("No such cached session exists for session id " + sessionId + ", re-instantiating validator.");
}
System.out.print(" Load FHIR v" + cliContext.getSv() + " from " + definitions);
ValidationEngine validator = new ValidationEngine.ValidationEngineBuilder().withVersion(cliContext.getSv()).withTimeTracker(tt).withUserAgent("fhir/validator").fromSource(definitions);
ValidationEngine validator = new ValidationEngine.ValidationEngineBuilder().withTHO(false).withVersion(cliContext.getSv()).withTimeTracker(tt).withUserAgent("fhir/validator").fromSource(definitions);
sessionId = sessionCache.cacheSession(validator);
FhirPublication ver = FhirPublication.fromCode(cliContext.getSv());
IgLoader igLoader = new IgLoader(validator.getPcm(), validator.getContext(), validator.getVersion(), validator.isDebug());
System.out.println(" - " + validator.getContext().countAllCaches() + " resources (" + tt.milestone() + ")");
IgLoader igLoader = new IgLoader(validator.getPcm(), validator.getContext(), validator.getVersion(), validator.isDebug());
igLoader.loadIg(validator.getIgs(), validator.getBinaries(), "hl7.terminology", false);
System.out.print(" Load R5 Extensions");
R5ExtensionsLoader r5e = new R5ExtensionsLoader(validator.getPcm(), validator.getContext());

View File

@ -19,7 +19,7 @@
<properties>
<hapi_fhir_version>5.4.0</hapi_fhir_version>
<validator_test_case_version>1.1.125</validator_test_case_version>
<validator_test_case_version>1.1.126-SNAPSHOT</validator_test_case_version>
<junit_jupiter_version>5.7.1</junit_jupiter_version>
<junit_platform_launcher_version>1.8.2</junit_platform_launcher_version>
<maven_surefire_version>3.0.0-M5</maven_surefire_version>