honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Performance improvements for validation, including -watch engine

This commit is contained in:
Grahame Grieve 2023-06-17 09:40:34 +10:00
parent 2d0dcbb803
commit 99895e7a97
17 changed files with 346 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/ExtensionExtractor.java
View File

@ -40,12 +40,12 @@ public class ExtensionExtractor {
Set<String> ids = new HashSet<>();
FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true);
NpmPackage r5 = pcm.loadPackage("hl7.fhir.r5.core", "current");
CanonicalResourceManager<CodeSystem> cslist = new CanonicalResourceManager<CodeSystem>(true);
CanonicalResourceManager<CodeSystem> cslist = new CanonicalResourceManager<CodeSystem>(true, false);
for (String r : r5.listResources("CodeSystem")) {
CodeSystem cs = (CodeSystem) new JsonParser().parse(r5.load(r));
cslist.see(cs, null);
}
CanonicalResourceManager<ValueSet> vslist = new CanonicalResourceManager<ValueSet>(true);
CanonicalResourceManager<ValueSet> vslist = new CanonicalResourceManager<ValueSet>(true, false);
for (String r : r5.listResources("ValueSet")) {
ValueSet vs = (ValueSet) new JsonParser().parse(r5.load(r));
vslist.see(vs, null);

16
org.hl7.fhir.core.generator/src/org/hl7/fhir/core/generator/engine/Definitions.java
View File

@ -17,15 +17,15 @@ import org.hl7.fhir.utilities.Utilities;
public class Definitions {
private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<ValueSet> valuesets = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<ConceptMap> conceptMaps = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<ValueSet> valuesets = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<ConceptMap> conceptMaps = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<CapabilityStatement> statements = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<SearchParameter> searchParams = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<CompartmentDefinition> compartments = new CanonicalResourceManager<>(true);
private CanonicalResourceManager<CapabilityStatement> statements = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<SearchParameter> searchParams = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<CompartmentDefinition> compartments = new CanonicalResourceManager<>(true, false);
public CanonicalResourceManager<CodeSystem> getCodeSystems() {

41
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java
View File

@ -210,27 +210,28 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
protected String version; // although the internal resources are all R5, the version of FHIR they describe may not be
protected TerminologyClientContext tcc = new TerminologyClientContext();
private boolean minimalMemory = false;
private Map<String, Map<String, ResourceProxy>> allResourcesById = new HashMap<String, Map<String, ResourceProxy>>();
// all maps are to the full URI
private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<CodeSystem>(false);
private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<CodeSystem>(false, minimalMemory);
private final Set<String> supportedCodeSystems = new HashSet<String>();
private final Set<String> unsupportedCodeSystems = new HashSet<String>(); // know that the terminology server doesn't support them
private CanonicalResourceManager<ValueSet> valueSets = new CanonicalResourceManager<ValueSet>(false);
private CanonicalResourceManager<ConceptMap> maps = new CanonicalResourceManager<ConceptMap>(false);
protected CanonicalResourceManager<StructureMap> transforms = new CanonicalResourceManager<StructureMap>(false);
private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<StructureDefinition>(false);
private final CanonicalResourceManager<Measure> measures = new CanonicalResourceManager<Measure>(false);
private final CanonicalResourceManager<Library> libraries = new CanonicalResourceManager<Library>(false);
private CanonicalResourceManager<ImplementationGuide> guides = new CanonicalResourceManager<ImplementationGuide>(false);
private final CanonicalResourceManager<CapabilityStatement> capstmts = new CanonicalResourceManager<CapabilityStatement>(false);
private final CanonicalResourceManager<SearchParameter> searchParameters = new CanonicalResourceManager<SearchParameter>(false);
private final CanonicalResourceManager<Questionnaire> questionnaires = new CanonicalResourceManager<Questionnaire>(false);
private final CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<OperationDefinition>(false);
private final CanonicalResourceManager<PlanDefinition> plans = new CanonicalResourceManager<PlanDefinition>(false);
private final CanonicalResourceManager<ActorDefinition> actors = new CanonicalResourceManager<ActorDefinition>(false);
private final CanonicalResourceManager<Requirements> requirements = new CanonicalResourceManager<Requirements>(false);
private final CanonicalResourceManager<NamingSystem> systems = new CanonicalResourceManager<NamingSystem>(false);
private CanonicalResourceManager<ValueSet> valueSets = new CanonicalResourceManager<ValueSet>(false, minimalMemory);
private CanonicalResourceManager<ConceptMap> maps = new CanonicalResourceManager<ConceptMap>(false, minimalMemory);
protected CanonicalResourceManager<StructureMap> transforms = new CanonicalResourceManager<StructureMap>(false, minimalMemory);
private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<StructureDefinition>(false, minimalMemory);
private final CanonicalResourceManager<Measure> measures = new CanonicalResourceManager<Measure>(false, minimalMemory);
private final CanonicalResourceManager<Library> libraries = new CanonicalResourceManager<Library>(false, minimalMemory);
private CanonicalResourceManager<ImplementationGuide> guides = new CanonicalResourceManager<ImplementationGuide>(false, minimalMemory);
private final CanonicalResourceManager<CapabilityStatement> capstmts = new CanonicalResourceManager<CapabilityStatement>(false, minimalMemory);
private final CanonicalResourceManager<SearchParameter> searchParameters = new CanonicalResourceManager<SearchParameter>(false, minimalMemory);
private final CanonicalResourceManager<Questionnaire> questionnaires = new CanonicalResourceManager<Questionnaire>(false, minimalMemory);
private final CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<OperationDefinition>(false, minimalMemory);
private final CanonicalResourceManager<PlanDefinition> plans = new CanonicalResourceManager<PlanDefinition>(false, minimalMemory);
private final CanonicalResourceManager<ActorDefinition> actors = new CanonicalResourceManager<ActorDefinition>(false, minimalMemory);
private final CanonicalResourceManager<Requirements> requirements = new CanonicalResourceManager<Requirements>(false, minimalMemory);
private final CanonicalResourceManager<NamingSystem> systems = new CanonicalResourceManager<NamingSystem>(false, minimalMemory);
private Map<String, NamingSystem> systemUrlMap;
@ -2343,23 +2344,25 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
return binaries.get(binaryKey);
}
public void finishLoading() {
public void finishLoading(boolean genSnapshots) {
if (!hasResource(StructureDefinition.class, "http://hl7.org/fhir/StructureDefinition/Base")) {
cacheResource(ProfileUtilities.makeBaseDefinition(version));
}
if(genSnapshots) {
for (StructureDefinition sd : listStructures()) {
try {
if (sd.getSnapshot().isEmpty()) {
new ContextUtilities(this).generateSnapshot(sd);
// new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path("[tmp]", "snapshot", tail(sd.getUrl())+".xml")), sd);
// new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path("[tmp]", "snapshot", tail(sd.getUrl())+".xml")), sd);
}
} catch (Exception e) {
System.out.println("Unable to generate snapshot for "+tail(sd.getUrl()) +" from "+tail(sd.getBaseDefinition())+" because "+e.getMessage());
System.out.println("Unable to generate snapshot @1 for "+tail(sd.getUrl()) +" from "+tail(sd.getBaseDefinition())+" because "+e.getMessage());
if (logger.isDebugLogging()) {
e.printStackTrace();
}
}
}
}
codeSystems.setVersion(version);
valueSets.setVersion(version);
maps.setVersion(version);

23
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/CanonicalResourceManager.java
View File

@ -32,6 +32,7 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
private String version;
private String supplements;
private CanonicalResource resource;
private boolean hacked;
public CanonicalResourceProxy(String type, String id, String url, String version, String supplements) {
super();
@ -77,6 +78,9 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
public CanonicalResource getResource() throws FHIRException {
if (resource == null) {
resource = loadResource();
if (hacked) {
resource.setUrl(url).setVersion(version);
}
if (resource instanceof CodeSystem) {
CodeSystemUtilities.crossLinkCodeSystem((CodeSystem) resource);
}
@ -98,7 +102,7 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
public void hack(String url, String version) {
this.url = url;
this.version = version;
getResource().setUrl(url).setVersion(version);
this.hacked = true;
}
}
@ -200,18 +204,25 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
}
}
private boolean minimalMemory;
private boolean enforceUniqueId;
private List<CachedCanonicalResource<T>> list = new ArrayList<>();
private Map<String, List<CachedCanonicalResource<T>>> listForId = new HashMap<>();
private Map<String, List<CachedCanonicalResource<T>>> listForUrl = new HashMap<>();
private Map<String, CachedCanonicalResource<T>> map = new HashMap<>();
private Map<String, List<CachedCanonicalResource<T>>> supplements = new HashMap<>(); // general index based on CodeSystem.supplements
private Map<String, List<CachedCanonicalResource<T>>> listForId;
private Map<String, List<CachedCanonicalResource<T>>> listForUrl;
private Map<String, CachedCanonicalResource<T>> map;
private Map<String, List<CachedCanonicalResource<T>>> supplements; // general index based on CodeSystem.supplements
private String version; // for debugging purposes
public CanonicalResourceManager(boolean enforceUniqueId) {
public CanonicalResourceManager(boolean enforceUniqueId, boolean minimalMemory) {
super();
this.enforceUniqueId = enforceUniqueId;
this.minimalMemory = minimalMemory;
list = new ArrayList<>();
listForId = new HashMap<>();
listForUrl = new HashMap<>();
map = new HashMap<>();
supplements = new HashMap<>(); // general index based on CodeSystem.supplements
}

14
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java
View File

@ -242,12 +242,12 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
return build(context);
}
public SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader) throws IOException, FHIRException {
public SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader, boolean genSnapshots) throws IOException, FHIRException {
SimpleWorkerContext context = getSimpleWorkerContextInstance();
context.setAllowLoadingDuplicates(allowLoadingDuplicates);
context.version = pi.getNpm().asString("version");
context.loadFromPackage(pi, loader);
context.finishLoading();
context.finishLoading(genSnapshots);
return build(context);
}
@ -667,7 +667,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
new ContextUtilities(this).generateSnapshot(p);
} catch (Exception e) {
// not sure what to do in this case?
System.out.println("Unable to generate snapshot for "+uri+": "+e.getMessage());
System.out.println("Unable to generate snapshot @3 for "+uri+": "+e.getMessage());
if (logger.isDebugLogging()) {
e.printStackTrace();
}
@ -676,6 +676,12 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
return r;
}
@Override
public <T extends Resource> T fetchResourceRaw(Class<T> class_, String uri) {
T r = super.fetchResource(class_, uri);
return r;
}
@Override
public <T extends Resource> T fetchResource(Class<T> class_, String uri, Resource source) {
T r = super.fetchResource(class_, uri, source);
@ -697,7 +703,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
}
} catch (Exception e) {
// not sure what to do in this case?
System.out.println("Unable to generate snapshot for "+p.getVersionedUrl()+": "+e.getMessage());
System.out.println("Unable to generate snapshot @4 for "+p.getVersionedUrl()+": "+e.getMessage());
if (logger.isDebugLogging()) {
e.printStackTrace();
}

2
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/ParserBase.java
View File

@ -161,7 +161,7 @@ public abstract class ParserBase {
logError(ValidationMessage.NO_RULE_DATE, line, col, name, IssueType.STRUCTURE, context.formatMessage(I18nConstants.THIS_CANNOT_BE_PARSED_AS_A_FHIR_OBJECT_NO_NAME), IssueSeverity.FATAL);
return null;
}
for (StructureDefinition sd : new ContextUtilities(context).allStructures()) {
for (StructureDefinition sd : context.fetchResourcesByType(StructureDefinition.class)) {
if (sd.getDerivation() == TypeDerivationRule.SPECIALIZATION && !sd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition/de-")) {
if(name.equals(sd.getType()) && (ns == null || ns.equals(FormatUtilities.FHIR_NS)) && !ToolingExtensions.hasExtension(sd, "http://hl7.org/fhir/StructureDefinition/elementdefinition-namespace"))
return sd;

2
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java
View File

@ -126,7 +126,7 @@ public class TestingUtilities extends BaseTestingUtilities {
public static SimpleWorkerContext getWorkerContext(NpmPackage npmPackage, IWorkerContext.IContextResourceLoader loader) throws Exception {
SimpleWorkerContext swc = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).withUserAgent(TestConstants.USER_AGENT)
.withTerminologyCachePath(getTerminologyCacheDirectory()).fromPackage(npmPackage, loader);
.withTerminologyCachePath(getTerminologyCacheDirectory()).fromPackage(npmPackage, loader, true);
TerminologyCache.setCacheErrors(true);
return swc;
}

36
org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CanonicalResourceManagerTests.java
View File

@ -31,7 +31,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleNoVersion() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet();
vs.setId("2345");
vs.setUrl("http://url/ValueSet/234");
@ -65,7 +65,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersion() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet();
vs.setId("2345");
vs.setUrl("http://url/ValueSet/234");
@ -85,7 +85,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersionNotSemVer() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet();
vs.setId("2345");
vs.setUrl("http://url/ValueSet/234");
@ -103,7 +103,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithDuplicateIds1() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -183,7 +183,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithDuplicateIds2() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -246,7 +246,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersions1() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -318,7 +318,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersions2() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -390,7 +390,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testUTG1() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet();
vs1.setId("234");
vs1.setUrl("http://terminology.hl7.org/ValueSet/234");
@ -418,7 +418,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testUTG2() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet();
vs1.setId("234");
vs1.setUrl("http://terminology.hl7.org/ValueSet/234");
@ -444,7 +444,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleNoVersionDeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet();
vs.setId("2345");
vs.setUrl("http://url/ValueSet/234");
@ -479,7 +479,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersionDeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet();
vs.setId("2345");
vs.setUrl("http://url/ValueSet/234");
@ -500,7 +500,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersionNotSemVerDeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet();
vs.setId("2345");
vs.setUrl("http://url/ValueSet/234");
@ -519,7 +519,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithDuplicateIds1DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -601,7 +601,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithDuplicateIds2DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -666,7 +666,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersions1DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -740,7 +740,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSingleWithVersions2DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -815,7 +815,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testPackageSpecificResolution1() {
// we add 2 canonicals to the cache with the same identification, but different package information
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false);
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet();
vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234");
@ -856,7 +856,7 @@ public class CanonicalResourceManagerTests {
@Test
public void testSupplements() {
CanonicalResourceManager<CodeSystem> mrm = new CanonicalResourceManager<>(true);
CanonicalResourceManager<CodeSystem> mrm = new CanonicalResourceManager<>(true, false);
CodeSystem csb1 = new CodeSystem();
csb1.setId("2345");
csb1.setUrl("http://url/CodeSystem/234");

9
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/Content.java
View File

@ -7,8 +7,6 @@ public class Content {
private byte[] focus = null;
private Manager.FhirFormat cntType = null;
public byte[] getFocus() {
return focus;
}
@ -21,5 +19,12 @@ public class Content {
public void setCntType(Manager.FhirFormat cntType) {
this.cntType = cntType;
}
public String getExampleFileName() {
if (cntType != null) {
return "file."+cntType.getExtension();
} else {
return "file.bin";
}
}
}

18
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java
View File

@ -39,12 +39,14 @@ import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
import org.hl7.fhir.utilities.npm.NpmPackage;
import org.hl7.fhir.utilities.turtle.Turtle;
import org.hl7.fhir.validation.ValidationEngine.IValidationEngineLoader;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.utils.Common;
import org.hl7.fhir.validation.cli.utils.VersionSourceInformation;
import lombok.Getter;
public class IgLoader {
public class IgLoader implements IValidationEngineLoader {
private static final String[] IGNORED_EXTENSIONS = {"md", "css", "js", "png", "gif", "jpg", "html", "tgz", "pack", "zip"};
private static final String[] EXEMPT_FILES = {"spec.internals", "version.info", "schematron.zip", "package.json"};
@ -301,11 +303,11 @@ public class IgLoader {
}
public void scanForVersions(List<String> sources, VersionSourceInformation versions) throws FHIRException, IOException {
List<String> refs = new ArrayList<String>();
List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(sources, refs, context);
for (String ref : refs) {
Content cnt = loadContent(ref, "validate", false);
scanForFhirVersion(versions, ref, cnt.getFocus());
for (SourceFile ref : refs) {
Content cnt = loadContent(ref.getRef(), "validate", false);
scanForFhirVersion(versions, ref.getRef(), cnt.getFocus());
}
}
@ -848,4 +850,10 @@ public class IgLoader {
private void log(String s) {
if (isDebug()) System.out.println(s);
}
@Override
public void load(Content cnt) throws FHIRException, IOException {
Resource res = loadResourceByVersion(version, cnt.getFocus(), cnt.getExampleFileName());
context.cacheResource(res);
}
}

19
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/Scanner.java
View File

@ -37,6 +37,7 @@ import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.utilities.xhtml.XhtmlComposer;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.model.ScanOutputItem;
import org.hl7.fhir.validation.instance.InstanceValidator;
@ -75,22 +76,22 @@ public class Scanner {
}
protected List<ScanOutputItem> validateScan(List<String> sources, Set<String> guides) throws FHIRException, IOException, EOperationOutcome {
List<String> refs = new ArrayList<>();
List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(sources, refs, getContext());
List<ScanOutputItem> res = new ArrayList<>();
for (String ref : refs) {
Content cnt = getIgLoader().loadContent(ref, "validate", false);
for (SourceFile ref : refs) {
Content cnt = getIgLoader().loadContent(ref.getRef(), "validate", false);
List<ValidationMessage> messages = new ArrayList<>();
Element e = null;
try {
System.out.println("Validate " + ref);
messages.clear();
e = getValidator().validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType());
res.add(new ScanOutputItem(ref, null, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
res.add(new ScanOutputItem(ref.getRef(), null, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
} catch (Exception ex) {
res.add(new ScanOutputItem(ref, null, null, exceptionToOutcome(ex)));
res.add(new ScanOutputItem(ref.getRef(), null, null, exceptionToOutcome(ex)));
}
if (e != null) {
String rt = e.fhirType();
@ -104,9 +105,9 @@ public class Scanner {
System.out.println("Validate " + ref + " against " + ig.getUrl());
messages.clear();
getValidator().validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType(), url);
res.add(new ScanOutputItem(ref, ig, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
res.add(new ScanOutputItem(ref.getRef(), ig, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
} catch (Exception ex) {
res.add(new ScanOutputItem(ref, ig, null, exceptionToOutcome(ex)));
res.add(new ScanOutputItem(ref.getRef(), ig, null, exceptionToOutcome(ex)));
}
}
Set<String> done = new HashSet<>();
@ -118,9 +119,9 @@ public class Scanner {
System.out.println("Validate " + ref + " against " + sd.getUrl());
messages.clear();
validator.validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType(), Collections.singletonList(sd));
res.add(new ScanOutputItem(ref, ig, sd, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
res.add(new ScanOutputItem(ref.getRef(), ig, sd, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
} catch (Exception ex) {
res.add(new ScanOutputItem(ref, ig, sd, exceptionToOutcome(ex)));
res.add(new ScanOutputItem(ref.getRef(), ig, sd, exceptionToOutcome(ex)));
}
}
}

70
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java
View File

@ -88,6 +88,8 @@ import org.hl7.fhir.utilities.settings.FhirSettings;
import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.utilities.xhtml.XhtmlComposer;
import org.hl7.fhir.validation.BaseValidator.ValidationControl;
import org.hl7.fhir.validation.ValidationEngine.IValidationEngineLoader;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.model.HtmlInMarkdownCheck;
import org.hl7.fhir.validation.cli.services.IPackageInstaller;
import org.hl7.fhir.validation.cli.utils.ProfileLoader;
@ -176,6 +178,13 @@ POSSIBILITY OF SUCH DAMAGE.
@Accessors(chain = true)
public class ValidationEngine implements IValidatorResourceFetcher, IValidationPolicyAdvisor, IPackageInstaller, IWorkerContextManager.IPackageLoadingTracker {
public interface IValidationEngineLoader {
void load(Content cnt) throws FHIRException, IOException;
}
@Getter @Setter private SimpleWorkerContext context;
@Getter @Setter private Map<String, byte[]> binaries = new HashMap<>();
@Getter @Setter private boolean doNative;
@ -423,7 +432,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
if (userAgent != null) {
contextBuilder.withUserAgent(userAgent);
}
context = contextBuilder.fromPackage(npm, ValidatorUtils.loaderForVersion(version));
context = contextBuilder.fromPackage(npm, ValidatorUtils.loaderForVersion(version), false);
} else {
Map<String, byte[]> source = igLoader.loadIgSource(src, recursive, true);
if (version == null) {
@ -543,28 +552,54 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
return list;
}
public OperationOutcome validate(String source, List<String> profiles) throws FHIRException, IOException {
public OperationOutcome validate(String source, List<String> profiles, IValidationEngineLoader loader, boolean all) throws FHIRException, IOException {
List<String> l = new ArrayList<String>();
List<SourceFile> refs = new ArrayList<>();
l.add(source);
return (OperationOutcome) validate(l, profiles, null);
return (OperationOutcome) validate(l, profiles, refs, null, loader, all);
}
public Resource validate(List<String> sources, List<String> profiles, List<ValidationRecord> record) throws FHIRException, IOException {
if (profiles.size() > 0) {
System.out.println(" Profiles: " + profiles);
}
List<String> refs = new ArrayList<String>();
public Resource validate(List<String> sources, List<String> profiles, List<SourceFile> refs, List<ValidationRecord> record, IValidationEngineLoader loader, boolean all) throws FHIRException, IOException {
boolean asBundle = ValidatorUtils.parseSources(sources, refs, context);
Bundle results = new Bundle();
results.setType(Bundle.BundleType.COLLECTION);
for (String ref : refs) {
boolean found = false;
for (SourceFile ref : refs) {
if (ref.isProcess()) {
found = true;
}
}
if (!found) {
return null;
}
// round one: try to read them all natively
// Ignore if it fails.The purpose of this is to make dependencies
// available for other resources to depend on. if it fails to load, there'll be an error if there's
// something that should've been loaded
for (SourceFile ref : refs) {
if (ref.isProcess() || all) {
ref.setCnt(igLoader.loadContent(ref.getRef(), "validate", false));
if (loader != null) {
try {
loader.load(ref.getCnt());
} catch (Throwable t) {
System.out.println(t.getMessage());
}
}
}
}
for (SourceFile ref : refs) {
if (ref.isProcess() || all) {
TimeTracker.Session tts = context.clock().start("validation");
context.clock().milestone();
System.out.println(" Validate " + ref);
Content cnt = igLoader.loadContent(ref, "validate", false);
try {
OperationOutcome outcome = validate(ref, cnt.getFocus(), cnt.getCntType(), profiles, record);
ToolingExtensions.addStringExtension(outcome, ToolingExtensions.EXT_OO_FILE, ref);
OperationOutcome outcome = validate(ref.getRef(), ref.getCnt().getFocus(), ref.getCnt().getCntType(), profiles, record);
ToolingExtensions.addStringExtension(outcome, ToolingExtensions.EXT_OO_FILE, ref.getRef());
System.out.println(" " + context.clock().milestone());
results.addEntry().setResource(outcome);
tts.end();
@ -573,6 +608,8 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
tts.end();
throw new FHIRException(e);
}
ref.setProcess(false);
}
}
if (asBundle)
return results;
@ -598,8 +635,17 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
System.out.println(location + ": " + validator.reportTimes());
}
if (record != null) {
boolean found = false;
for (ValidationRecord t : record) {
if (t.getLocation().equals(location)) {
found = true;
t.setMessages(messages);
}
}
if (!found) {
record.add(new ValidationRecord(location, messages));
}
}
return ValidatorUtils.messagesToOutcome(messages, context, fhirPathEngine);
}

5
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationRecord.java
View File

@ -46,4 +46,9 @@ public class ValidationRecord {
return info;
}
public void setMessages(List<ValidationMessage> messages) {
this.messages = messages;
}
}

2
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java
View File

@ -317,8 +317,6 @@ public class ValidatorCli {
}
System.out.println("Loading");
// Comment this out because definitions filename doesn't necessarily contain version (and many not even be 14 characters long).
// Version gets spit out a couple of lines later after we've loaded the context
String definitions = "dev".equals(cliContext.getSv()) ? "hl7.fhir.r5.core#current" : VersionUtilities.packageForVersion(cliContext.getSv()) + "#" + VersionUtilities.getCurrentVersion(cliContext.getSv());
ValidationEngine validator = validationService.initializeValidator(cliContext, definitions, tt);
tts.end();

68
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorUtils.java
View File

@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -43,6 +44,33 @@ import org.xml.sax.SAXException;
//TODO find a home for these and clean it up
public class ValidatorUtils {
public static class SourceFile {
private String ref;
private long date;
private boolean process;
private Content cnt;
public boolean isProcess() {
return process;
}
public void setProcess(boolean process) {
this.process = process;
this.cnt = null;
}
public String getRef() {
return ref;
}
public long getDate() {
return date;
}
public Content getCnt() {
return cnt;
}
public void setCnt(Content cnt) {
this.cnt = cnt;
}
}
protected static void grabNatives(Map<String, byte[]> source, Map<String, byte[]> binaries, String prefix) {
for (Map.Entry<String, byte[]> e : source.entrySet()) {
if (e.getKey().endsWith(".zip"))
@ -129,14 +157,15 @@ public class ValidatorUtils {
*
* @return {@link Boolean#TRUE} if more than one reference is found.
*/
static boolean extractReferences(String name, List<String> refs, SimpleWorkerContext context) throws IOException {
static boolean extractReferences(String name, List<SourceFile> refs, SimpleWorkerContext context) throws IOException {
if (Common.isNetworkPath(name)) {
refs.add(name);
SourceFile src = addSourceFile(refs, name);
src.date = Long.MAX_VALUE;
} else if (Common.isWildcardPath(name)) {
AsteriskFilter filter = new AsteriskFilter(name);
File[] files = new File(filter.getDir()).listFiles(filter);
for (File file : files) {
refs.add(file.getPath());
addSourceFile(refs, file);
}
} else {
File file = new File(name);
@ -150,24 +179,49 @@ public class ValidatorUtils {
}
if (file.isFile()) {
refs.add(name);
addSourceFile(refs, file);
} else {
for (int i = 0; i < file.listFiles().length; i++) {
File[] fileList = file.listFiles();
if (fileList[i].isFile())
refs.add(fileList[i].getPath());
if (fileList[i].isFile()) {
if (!Utilities.isIgnorableFile(fileList[i])) {
addSourceFile(refs, fileList[i]);
}
}
}
}
}
return refs.size() > 1;
}
private static SourceFile addSourceFile(List<SourceFile> refs, File file) {
SourceFile src = addSourceFile(refs, file.getPath());
Long l = file.lastModified();
if (src.date != l) {
src.process = true;
}
src.date = l;
return src;
}
private static SourceFile addSourceFile(List<SourceFile> refs, String path) {
for (SourceFile t : refs) {
if (t.ref.equals(path)) {
return t;
}
}
SourceFile src = new SourceFile();
src.ref = path;
refs.add(src);
return src;
}
/**
* Iterates through the list of passed in sources, extracting all references and populated them in the passed in list.
*
* @return {@link Boolean#TRUE} if more than one reference is found.
*/
public static boolean parseSources(List<String> sources, List<String> refs, SimpleWorkerContext context) throws IOException {
public static boolean parseSources(List<String> sources, List<SourceFile> refs, SimpleWorkerContext context) throws IOException {
boolean multipleRefsFound = sources.size() > 1;
for (String source : sources) {
multipleRefsFound |= extractReferences(source, refs, context);

44
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidationService.java
View File

@ -65,8 +65,10 @@ import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.validation.Content;
import org.hl7.fhir.validation.IgLoader;
import org.hl7.fhir.validation.ValidationEngine;
import org.hl7.fhir.validation.ValidationEngine.IValidationEngineLoader;
import org.hl7.fhir.validation.ValidationRecord;
import org.hl7.fhir.validation.ValidatorUtils;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.model.CliContext;
import org.hl7.fhir.validation.cli.model.FileInfo;
import org.hl7.fhir.validation.cli.model.ValidationOutcome;
@ -142,9 +144,23 @@ public class ValidationService {
}
public void validateSources(CliContext cliContext, ValidationEngine validator) throws Exception {
long start = System.currentTimeMillis();
if (cliContext.getProfiles().size() > 0) {
System.out.println(" Profiles: " + cliContext.getProfiles());
}
ValidatorWatchMode watch = ValidatorWatchMode.NONE;
IgLoader igLoader = new IgLoader(validator.getPcm(), validator.getContext(), validator.getVersion());
List<ValidationRecord> records = new ArrayList<>();
Resource r = validator.validate(cliContext.getSources(), cliContext.getProfiles(), records);
List<SourceFile> refs = new ArrayList<>();
int ec = 0;
do {
long start = System.currentTimeMillis();
Resource r = validator.validate(cliContext.getSources(), cliContext.getProfiles(), refs, records, igLoader, watch == ValidatorWatchMode.ALL);
boolean statusNeeded = false;
if (r != null) {
statusNeeded = true;
MemoryMXBean mbean = ManagementFactory.getMemoryMXBean();
System.out.println("Done. " + validator.getContext().clock().report()+". Memory = "+Utilities.describeSize(mbean.getHeapMemoryUsage().getUsed()+mbean.getNonHeapMemoryUsage().getUsed()));
System.out.println();
@ -168,8 +184,6 @@ public class ValidationService {
renderer.setFolder(dir);
}
int ec = 0;
if (r instanceof Bundle) {
if (renderer.handlesBundleDirectly()) {
renderer.render((Bundle) r);
@ -202,6 +216,14 @@ public class ValidationService {
TextFile.stringToFile(html, cliContext.getHtmlOutput());
System.out.println("HTML Summary in " + cliContext.getHtmlOutput());
}
}
if (watch != ValidatorWatchMode.NONE) {
if (statusNeeded) {
System.out.println("Watching for changes");
}
Thread.sleep(1000);
}
} while (watch != ValidatorWatchMode.NONE);
System.exit(ec > 0 ? 1 : 0);
}
@ -552,11 +574,11 @@ public class ValidationService {
XLIFFProducer xliff = new XLIFFProducer(Utilities.path(dst));
JsonLangFileProducer jl = new JsonLangFileProducer(Utilities.path(dst));
List<String> refs = new ArrayList<String>();
List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(cliContext.getSources(), refs, validator.getContext());
for (String ref : refs) {
for (SourceFile ref : refs) {
System.out.println(" Extract Translations from " + ref);
org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref, "translate", false);
org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref.getRef(), "translate", false);
Element e = Manager.parseSingle(validator.getContext(), new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType());
LanguageProducerSession ps = po.startSession(e.fhirType()+"-"+e.getIdBase(), cliContext.getSrcLang());
LanguageProducerLanguageSession psl = ps.forLang(cliContext.getTgtLang());
@ -586,15 +608,15 @@ public class ValidationService {
loadTranslationSource(translations, input);
}
List<String> refs = new ArrayList<String>();
List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(cliContext.getSources(), refs, validator.getContext());
int t = 0;
for (String ref : refs) {
for (SourceFile ref : refs) {
System.out.println(" Inject Translations into " + ref);
org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref, "translate", false);
org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref.getRef(), "translate", false);
Element e = Manager.parseSingle(validator.getContext(), new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType());
t = t + new LanguageUtils(validator.getContext()).importFromTranslations(e, translations);
Manager.compose(validator.getContext(), e, new FileOutputStream(Utilities.path(dst, new File(ref).getName())), cnt.getCntType(),
Manager.compose(validator.getContext(), e, new FileOutputStream(Utilities.path(dst, new File(ref.getRef()).getName())), cnt.getCntType(),
OutputStyle.PRETTY, null);
}
System.out.println("Done - imported "+t+" translations into "+refs.size()+ " in "+dst);

8
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidatorWatchMode.java Normal file
View File

@ -0,0 +1,8 @@
package org.hl7.fhir.validation.cli.services;
public enum ValidatorWatchMode {
NONE, // just stop when validation is done
SINGLE, // when validation is done, watch the content that was validated, and revalidate anything that changes
ALL // when validation is done, watch the content that was validated, and revalidate everything if anything changes
}