honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Performance improvements for validation, including -watch engine

This commit is contained in:
Grahame Grieve 2023-06-17 09:40:34 +10:00
parent 2d0dcbb803
commit 99895e7a97
17 changed files with 346 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/ExtensionExtractor.java
View File

@ -40,12 +40,12 @@ public class ExtensionExtractor {
Set<String> ids = new HashSet<>(); Set<String> ids = new HashSet<>();
FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true); FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager(true);
NpmPackage r5 = pcm.loadPackage("hl7.fhir.r5.core", "current"); NpmPackage r5 = pcm.loadPackage("hl7.fhir.r5.core", "current");
CanonicalResourceManager<CodeSystem> cslist = new CanonicalResourceManager<CodeSystem>(true); CanonicalResourceManager<CodeSystem> cslist = new CanonicalResourceManager<CodeSystem>(true, false);
for (String r : r5.listResources("CodeSystem")) { for (String r : r5.listResources("CodeSystem")) {
CodeSystem cs = (CodeSystem) new JsonParser().parse(r5.load(r)); CodeSystem cs = (CodeSystem) new JsonParser().parse(r5.load(r));
cslist.see(cs, null); cslist.see(cs, null);
} }
CanonicalResourceManager<ValueSet> vslist = new CanonicalResourceManager<ValueSet>(true); CanonicalResourceManager<ValueSet> vslist = new CanonicalResourceManager<ValueSet>(true, false);
for (String r : r5.listResources("ValueSet")) { for (String r : r5.listResources("ValueSet")) {
ValueSet vs = (ValueSet) new JsonParser().parse(r5.load(r)); ValueSet vs = (ValueSet) new JsonParser().parse(r5.load(r));
vslist.see(vs, null); vslist.see(vs, null);

16
org.hl7.fhir.core.generator/src/org/hl7/fhir/core/generator/engine/Definitions.java
View File

@ -17,15 +17,15 @@ import org.hl7.fhir.utilities.Utilities;
public class Definitions { public class Definitions {
private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<>(true); private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<ValueSet> valuesets = new CanonicalResourceManager<>(true); private CanonicalResourceManager<ValueSet> valuesets = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<ConceptMap> conceptMaps = new CanonicalResourceManager<>(true); private CanonicalResourceManager<ConceptMap> conceptMaps = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<CapabilityStatement> statements = new CanonicalResourceManager<>(true); private CanonicalResourceManager<CapabilityStatement> statements = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<>(true); private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<>(true); private CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<SearchParameter> searchParams = new CanonicalResourceManager<>(true); private CanonicalResourceManager<SearchParameter> searchParams = new CanonicalResourceManager<>(true, false);
private CanonicalResourceManager<CompartmentDefinition> compartments = new CanonicalResourceManager<>(true); private CanonicalResourceManager<CompartmentDefinition> compartments = new CanonicalResourceManager<>(true, false);
public CanonicalResourceManager<CodeSystem> getCodeSystems() { public CanonicalResourceManager<CodeSystem> getCodeSystems() {

61
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java
View File

@ -210,27 +210,28 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
protected String version; // although the internal resources are all R5, the version of FHIR they describe may not be protected String version; // although the internal resources are all R5, the version of FHIR they describe may not be
protected TerminologyClientContext tcc = new TerminologyClientContext(); protected TerminologyClientContext tcc = new TerminologyClientContext();
private boolean minimalMemory = false;
private Map<String, Map<String, ResourceProxy>> allResourcesById = new HashMap<String, Map<String, ResourceProxy>>(); private Map<String, Map<String, ResourceProxy>> allResourcesById = new HashMap<String, Map<String, ResourceProxy>>();
// all maps are to the full URI // all maps are to the full URI
private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<CodeSystem>(false); private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<CodeSystem>(false, minimalMemory);
private final Set<String> supportedCodeSystems = new HashSet<String>(); private final Set<String> supportedCodeSystems = new HashSet<String>();
private final Set<String> unsupportedCodeSystems = new HashSet<String>(); // know that the terminology server doesn't support them private final Set<String> unsupportedCodeSystems = new HashSet<String>(); // know that the terminology server doesn't support them
private CanonicalResourceManager<ValueSet> valueSets = new CanonicalResourceManager<ValueSet>(false); private CanonicalResourceManager<ValueSet> valueSets = new CanonicalResourceManager<ValueSet>(false, minimalMemory);
private CanonicalResourceManager<ConceptMap> maps = new CanonicalResourceManager<ConceptMap>(false); private CanonicalResourceManager<ConceptMap> maps = new CanonicalResourceManager<ConceptMap>(false, minimalMemory);
protected CanonicalResourceManager<StructureMap> transforms = new CanonicalResourceManager<StructureMap>(false); protected CanonicalResourceManager<StructureMap> transforms = new CanonicalResourceManager<StructureMap>(false, minimalMemory);
private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<StructureDefinition>(false); private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<StructureDefinition>(false, minimalMemory);
private final CanonicalResourceManager<Measure> measures = new CanonicalResourceManager<Measure>(false); private final CanonicalResourceManager<Measure> measures = new CanonicalResourceManager<Measure>(false, minimalMemory);
private final CanonicalResourceManager<Library> libraries = new CanonicalResourceManager<Library>(false); private final CanonicalResourceManager<Library> libraries = new CanonicalResourceManager<Library>(false, minimalMemory);
private CanonicalResourceManager<ImplementationGuide> guides = new CanonicalResourceManager<ImplementationGuide>(false); private CanonicalResourceManager<ImplementationGuide> guides = new CanonicalResourceManager<ImplementationGuide>(false, minimalMemory);
private final CanonicalResourceManager<CapabilityStatement> capstmts = new CanonicalResourceManager<CapabilityStatement>(false); private final CanonicalResourceManager<CapabilityStatement> capstmts = new CanonicalResourceManager<CapabilityStatement>(false, minimalMemory);
private final CanonicalResourceManager<SearchParameter> searchParameters = new CanonicalResourceManager<SearchParameter>(false); private final CanonicalResourceManager<SearchParameter> searchParameters = new CanonicalResourceManager<SearchParameter>(false, minimalMemory);
private final CanonicalResourceManager<Questionnaire> questionnaires = new CanonicalResourceManager<Questionnaire>(false); private final CanonicalResourceManager<Questionnaire> questionnaires = new CanonicalResourceManager<Questionnaire>(false, minimalMemory);
private final CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<OperationDefinition>(false); private final CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<OperationDefinition>(false, minimalMemory);
private final CanonicalResourceManager<PlanDefinition> plans = new CanonicalResourceManager<PlanDefinition>(false); private final CanonicalResourceManager<PlanDefinition> plans = new CanonicalResourceManager<PlanDefinition>(false, minimalMemory);
private final CanonicalResourceManager<ActorDefinition> actors = new CanonicalResourceManager<ActorDefinition>(false); private final CanonicalResourceManager<ActorDefinition> actors = new CanonicalResourceManager<ActorDefinition>(false, minimalMemory);
private final CanonicalResourceManager<Requirements> requirements = new CanonicalResourceManager<Requirements>(false); private final CanonicalResourceManager<Requirements> requirements = new CanonicalResourceManager<Requirements>(false, minimalMemory);
private final CanonicalResourceManager<NamingSystem> systems = new CanonicalResourceManager<NamingSystem>(false); private final CanonicalResourceManager<NamingSystem> systems = new CanonicalResourceManager<NamingSystem>(false, minimalMemory);
private Map<String, NamingSystem> systemUrlMap; private Map<String, NamingSystem> systemUrlMap;
@ -2343,23 +2344,25 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte
return binaries.get(binaryKey); return binaries.get(binaryKey);
} }
public void finishLoading() { public void finishLoading(boolean genSnapshots) {
if (!hasResource(StructureDefinition.class, "http://hl7.org/fhir/StructureDefinition/Base")) { if (!hasResource(StructureDefinition.class, "http://hl7.org/fhir/StructureDefinition/Base")) {
cacheResource(ProfileUtilities.makeBaseDefinition(version)); cacheResource(ProfileUtilities.makeBaseDefinition(version));
} }
for (StructureDefinition sd : listStructures()) { if(genSnapshots) {
try { for (StructureDefinition sd : listStructures()) {
if (sd.getSnapshot().isEmpty()) { try {
new ContextUtilities(this).generateSnapshot(sd); if (sd.getSnapshot().isEmpty()) {
// new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path("[tmp]", "snapshot", tail(sd.getUrl())+".xml")), sd); new ContextUtilities(this).generateSnapshot(sd);
// new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path("[tmp]", "snapshot", tail(sd.getUrl())+".xml")), sd);
}
} catch (Exception e) {
System.out.println("Unable to generate snapshot @1 for "+tail(sd.getUrl()) +" from "+tail(sd.getBaseDefinition())+" because "+e.getMessage());
if (logger.isDebugLogging()) {
e.printStackTrace();
}
} }
} catch (Exception e) { }
System.out.println("Unable to generate snapshot for "+tail(sd.getUrl()) +" from "+tail(sd.getBaseDefinition())+" because "+e.getMessage()); }
if (logger.isDebugLogging()) {
e.printStackTrace();
}
}
}
codeSystems.setVersion(version); codeSystems.setVersion(version);
valueSets.setVersion(version); valueSets.setVersion(version);
maps.setVersion(version); maps.setVersion(version);

23
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/CanonicalResourceManager.java
View File

@ -32,6 +32,7 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
private String version; private String version;
private String supplements; private String supplements;
private CanonicalResource resource; private CanonicalResource resource;
private boolean hacked;
public CanonicalResourceProxy(String type, String id, String url, String version, String supplements) { public CanonicalResourceProxy(String type, String id, String url, String version, String supplements) {
super(); super();
@ -77,6 +78,9 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
public CanonicalResource getResource() throws FHIRException { public CanonicalResource getResource() throws FHIRException {
if (resource == null) { if (resource == null) {
resource = loadResource(); resource = loadResource();
if (hacked) {
resource.setUrl(url).setVersion(version);
}
if (resource instanceof CodeSystem) { if (resource instanceof CodeSystem) {
CodeSystemUtilities.crossLinkCodeSystem((CodeSystem) resource); CodeSystemUtilities.crossLinkCodeSystem((CodeSystem) resource);
} }
@ -98,7 +102,7 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
public void hack(String url, String version) { public void hack(String url, String version) {
this.url = url; this.url = url;
this.version = version; this.version = version;
getResource().setUrl(url).setVersion(version); this.hacked = true;
} }
} }
@ -200,18 +204,25 @@ public class CanonicalResourceManager<T extends CanonicalResource> {
} }
} }
private boolean minimalMemory;
private boolean enforceUniqueId; private boolean enforceUniqueId;
private List<CachedCanonicalResource<T>> list = new ArrayList<>(); private List<CachedCanonicalResource<T>> list = new ArrayList<>();
private Map<String, List<CachedCanonicalResource<T>>> listForId = new HashMap<>(); private Map<String, List<CachedCanonicalResource<T>>> listForId;
private Map<String, List<CachedCanonicalResource<T>>> listForUrl = new HashMap<>(); private Map<String, List<CachedCanonicalResource<T>>> listForUrl;
private Map<String, CachedCanonicalResource<T>> map = new HashMap<>(); private Map<String, CachedCanonicalResource<T>> map;
private Map<String, List<CachedCanonicalResource<T>>> supplements = new HashMap<>(); // general index based on CodeSystem.supplements private Map<String, List<CachedCanonicalResource<T>>> supplements; // general index based on CodeSystem.supplements
private String version; // for debugging purposes private String version; // for debugging purposes
public CanonicalResourceManager(boolean enforceUniqueId) { public CanonicalResourceManager(boolean enforceUniqueId, boolean minimalMemory) {
super(); super();
this.enforceUniqueId = enforceUniqueId; this.enforceUniqueId = enforceUniqueId;
this.minimalMemory = minimalMemory;
list = new ArrayList<>();
listForId = new HashMap<>();
listForUrl = new HashMap<>();
map = new HashMap<>();
supplements = new HashMap<>(); // general index based on CodeSystem.supplements
} }

14
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java
View File

@ -242,12 +242,12 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
return build(context); return build(context);
} }
public SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader) throws IOException, FHIRException { public SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader, boolean genSnapshots) throws IOException, FHIRException {
SimpleWorkerContext context = getSimpleWorkerContextInstance(); SimpleWorkerContext context = getSimpleWorkerContextInstance();
context.setAllowLoadingDuplicates(allowLoadingDuplicates); context.setAllowLoadingDuplicates(allowLoadingDuplicates);
context.version = pi.getNpm().asString("version"); context.version = pi.getNpm().asString("version");
context.loadFromPackage(pi, loader); context.loadFromPackage(pi, loader);
context.finishLoading(); context.finishLoading(genSnapshots);
return build(context); return build(context);
} }
@ -667,7 +667,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
new ContextUtilities(this).generateSnapshot(p); new ContextUtilities(this).generateSnapshot(p);
} catch (Exception e) { } catch (Exception e) {
// not sure what to do in this case? // not sure what to do in this case?
System.out.println("Unable to generate snapshot for "+uri+": "+e.getMessage()); System.out.println("Unable to generate snapshot @3 for "+uri+": "+e.getMessage());
if (logger.isDebugLogging()) { if (logger.isDebugLogging()) {
e.printStackTrace(); e.printStackTrace();
} }
@ -676,6 +676,12 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
return r; return r;
} }
@Override
public <T extends Resource> T fetchResourceRaw(Class<T> class_, String uri) {
T r = super.fetchResource(class_, uri);
return r;
}
@Override @Override
public <T extends Resource> T fetchResource(Class<T> class_, String uri, Resource source) { public <T extends Resource> T fetchResource(Class<T> class_, String uri, Resource source) {
T r = super.fetchResource(class_, uri, source); T r = super.fetchResource(class_, uri, source);
@ -697,7 +703,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
} }
} catch (Exception e) { } catch (Exception e) {
// not sure what to do in this case? // not sure what to do in this case?
System.out.println("Unable to generate snapshot for "+p.getVersionedUrl()+": "+e.getMessage()); System.out.println("Unable to generate snapshot @4 for "+p.getVersionedUrl()+": "+e.getMessage());
if (logger.isDebugLogging()) { if (logger.isDebugLogging()) {
e.printStackTrace(); e.printStackTrace();
} }

2
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/ParserBase.java
View File

@ -161,7 +161,7 @@ public abstract class ParserBase {
logError(ValidationMessage.NO_RULE_DATE, line, col, name, IssueType.STRUCTURE, context.formatMessage(I18nConstants.THIS_CANNOT_BE_PARSED_AS_A_FHIR_OBJECT_NO_NAME), IssueSeverity.FATAL); logError(ValidationMessage.NO_RULE_DATE, line, col, name, IssueType.STRUCTURE, context.formatMessage(I18nConstants.THIS_CANNOT_BE_PARSED_AS_A_FHIR_OBJECT_NO_NAME), IssueSeverity.FATAL);
return null; return null;
} }
for (StructureDefinition sd : new ContextUtilities(context).allStructures()) { for (StructureDefinition sd : context.fetchResourcesByType(StructureDefinition.class)) {
if (sd.getDerivation() == TypeDerivationRule.SPECIALIZATION && !sd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition/de-")) { if (sd.getDerivation() == TypeDerivationRule.SPECIALIZATION && !sd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition/de-")) {
if(name.equals(sd.getType()) && (ns == null || ns.equals(FormatUtilities.FHIR_NS)) && !ToolingExtensions.hasExtension(sd, "http://hl7.org/fhir/StructureDefinition/elementdefinition-namespace")) if(name.equals(sd.getType()) && (ns == null || ns.equals(FormatUtilities.FHIR_NS)) && !ToolingExtensions.hasExtension(sd, "http://hl7.org/fhir/StructureDefinition/elementdefinition-namespace"))
return sd; return sd;

2
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/test/utils/TestingUtilities.java
View File

@ -126,7 +126,7 @@ public class TestingUtilities extends BaseTestingUtilities {
public static SimpleWorkerContext getWorkerContext(NpmPackage npmPackage, IWorkerContext.IContextResourceLoader loader) throws Exception { public static SimpleWorkerContext getWorkerContext(NpmPackage npmPackage, IWorkerContext.IContextResourceLoader loader) throws Exception {
SimpleWorkerContext swc = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).withUserAgent(TestConstants.USER_AGENT) SimpleWorkerContext swc = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).withUserAgent(TestConstants.USER_AGENT)
.withTerminologyCachePath(getTerminologyCacheDirectory()).fromPackage(npmPackage, loader); .withTerminologyCachePath(getTerminologyCacheDirectory()).fromPackage(npmPackage, loader, true);
TerminologyCache.setCacheErrors(true); TerminologyCache.setCacheErrors(true);
return swc; return swc;
} }

36
org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/CanonicalResourceManagerTests.java
View File

@ -31,7 +31,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleNoVersion() { public void testSingleNoVersion() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet(); ValueSet vs = new ValueSet();
vs.setId("2345"); vs.setId("2345");
vs.setUrl("http://url/ValueSet/234"); vs.setUrl("http://url/ValueSet/234");
@ -65,7 +65,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersion() { public void testSingleWithVersion() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet(); ValueSet vs = new ValueSet();
vs.setId("2345"); vs.setId("2345");
vs.setUrl("http://url/ValueSet/234"); vs.setUrl("http://url/ValueSet/234");
@ -85,7 +85,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersionNotSemVer() { public void testSingleWithVersionNotSemVer() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet(); ValueSet vs = new ValueSet();
vs.setId("2345"); vs.setId("2345");
vs.setUrl("http://url/ValueSet/234"); vs.setUrl("http://url/ValueSet/234");
@ -103,7 +103,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithDuplicateIds1() { public void testSingleWithDuplicateIds1() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -183,7 +183,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithDuplicateIds2() { public void testSingleWithDuplicateIds2() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -246,7 +246,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersions1() { public void testSingleWithVersions1() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -318,7 +318,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersions2() { public void testSingleWithVersions2() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -390,7 +390,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testUTG1() { public void testUTG1() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("234"); vs1.setId("234");
vs1.setUrl("http://terminology.hl7.org/ValueSet/234"); vs1.setUrl("http://terminology.hl7.org/ValueSet/234");
@ -418,7 +418,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testUTG2() { public void testUTG2() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("234"); vs1.setId("234");
vs1.setUrl("http://terminology.hl7.org/ValueSet/234"); vs1.setUrl("http://terminology.hl7.org/ValueSet/234");
@ -444,7 +444,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleNoVersionDeferredLoad() { public void testSingleNoVersionDeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet(); ValueSet vs = new ValueSet();
vs.setId("2345"); vs.setId("2345");
vs.setUrl("http://url/ValueSet/234"); vs.setUrl("http://url/ValueSet/234");
@ -479,7 +479,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersionDeferredLoad() { public void testSingleWithVersionDeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet(); ValueSet vs = new ValueSet();
vs.setId("2345"); vs.setId("2345");
vs.setUrl("http://url/ValueSet/234"); vs.setUrl("http://url/ValueSet/234");
@ -500,7 +500,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersionNotSemVerDeferredLoad() { public void testSingleWithVersionNotSemVerDeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs = new ValueSet(); ValueSet vs = new ValueSet();
vs.setId("2345"); vs.setId("2345");
vs.setUrl("http://url/ValueSet/234"); vs.setUrl("http://url/ValueSet/234");
@ -519,7 +519,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithDuplicateIds1DeferredLoad() { public void testSingleWithDuplicateIds1DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -601,7 +601,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithDuplicateIds2DeferredLoad() { public void testSingleWithDuplicateIds2DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -666,7 +666,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersions1DeferredLoad() { public void testSingleWithVersions1DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -740,7 +740,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSingleWithVersions2DeferredLoad() { public void testSingleWithVersions2DeferredLoad() {
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(true, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -815,7 +815,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testPackageSpecificResolution1() { public void testPackageSpecificResolution1() {
// we add 2 canonicals to the cache with the same identification, but different package information // we add 2 canonicals to the cache with the same identification, but different package information
CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false); CanonicalResourceManager<ValueSet> mrm = new CanonicalResourceManager<>(false, false);
ValueSet vs1 = new ValueSet(); ValueSet vs1 = new ValueSet();
vs1.setId("2345"); vs1.setId("2345");
vs1.setUrl("http://url/ValueSet/234"); vs1.setUrl("http://url/ValueSet/234");
@ -856,7 +856,7 @@ public class CanonicalResourceManagerTests {
@Test @Test
public void testSupplements() { public void testSupplements() {
CanonicalResourceManager<CodeSystem> mrm = new CanonicalResourceManager<>(true); CanonicalResourceManager<CodeSystem> mrm = new CanonicalResourceManager<>(true, false);
CodeSystem csb1 = new CodeSystem(); CodeSystem csb1 = new CodeSystem();
csb1.setId("2345"); csb1.setId("2345");
csb1.setUrl("http://url/CodeSystem/234"); csb1.setUrl("http://url/CodeSystem/234");

9
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/Content.java
View File

@ -7,8 +7,6 @@ public class Content {
private byte[] focus = null; private byte[] focus = null;
private Manager.FhirFormat cntType = null; private Manager.FhirFormat cntType = null;
public byte[] getFocus() { public byte[] getFocus() {
return focus; return focus;
} }
@ -21,5 +19,12 @@ public class Content {
public void setCntType(Manager.FhirFormat cntType) { public void setCntType(Manager.FhirFormat cntType) {
this.cntType = cntType; this.cntType = cntType;
} }
public String getExampleFileName() {
if (cntType != null) {
return "file."+cntType.getExtension();
} else {
return "file.bin";
}
}
} }

18
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java
View File

@ -39,12 +39,14 @@ import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager; import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
import org.hl7.fhir.utilities.npm.NpmPackage; import org.hl7.fhir.utilities.npm.NpmPackage;
import org.hl7.fhir.utilities.turtle.Turtle; import org.hl7.fhir.utilities.turtle.Turtle;
import org.hl7.fhir.validation.ValidationEngine.IValidationEngineLoader;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.utils.Common; import org.hl7.fhir.validation.cli.utils.Common;
import org.hl7.fhir.validation.cli.utils.VersionSourceInformation; import org.hl7.fhir.validation.cli.utils.VersionSourceInformation;
import lombok.Getter; import lombok.Getter;
public class IgLoader { public class IgLoader implements IValidationEngineLoader {
private static final String[] IGNORED_EXTENSIONS = {"md", "css", "js", "png", "gif", "jpg", "html", "tgz", "pack", "zip"}; private static final String[] IGNORED_EXTENSIONS = {"md", "css", "js", "png", "gif", "jpg", "html", "tgz", "pack", "zip"};
private static final String[] EXEMPT_FILES = {"spec.internals", "version.info", "schematron.zip", "package.json"}; private static final String[] EXEMPT_FILES = {"spec.internals", "version.info", "schematron.zip", "package.json"};
@ -301,11 +303,11 @@ public class IgLoader {
} }
public void scanForVersions(List<String> sources, VersionSourceInformation versions) throws FHIRException, IOException { public void scanForVersions(List<String> sources, VersionSourceInformation versions) throws FHIRException, IOException {
List<String> refs = new ArrayList<String>(); List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(sources, refs, context); ValidatorUtils.parseSources(sources, refs, context);
for (String ref : refs) { for (SourceFile ref : refs) {
Content cnt = loadContent(ref, "validate", false); Content cnt = loadContent(ref.getRef(), "validate", false);
scanForFhirVersion(versions, ref, cnt.getFocus()); scanForFhirVersion(versions, ref.getRef(), cnt.getFocus());
} }
} }
@ -848,4 +850,10 @@ public class IgLoader {
private void log(String s) { private void log(String s) {
if (isDebug()) System.out.println(s); if (isDebug()) System.out.println(s);
} }
@Override
public void load(Content cnt) throws FHIRException, IOException {
Resource res = loadResourceByVersion(version, cnt.getFocus(), cnt.getExampleFileName());
context.cacheResource(res);
}
} }

19
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/Scanner.java
View File

@ -37,6 +37,7 @@ import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.utilities.xhtml.XhtmlComposer; import org.hl7.fhir.utilities.xhtml.XhtmlComposer;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.model.ScanOutputItem; import org.hl7.fhir.validation.cli.model.ScanOutputItem;
import org.hl7.fhir.validation.instance.InstanceValidator; import org.hl7.fhir.validation.instance.InstanceValidator;
@ -75,22 +76,22 @@ public class Scanner {
} }
protected List<ScanOutputItem> validateScan(List<String> sources, Set<String> guides) throws FHIRException, IOException, EOperationOutcome { protected List<ScanOutputItem> validateScan(List<String> sources, Set<String> guides) throws FHIRException, IOException, EOperationOutcome {
List<String> refs = new ArrayList<>(); List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(sources, refs, getContext()); ValidatorUtils.parseSources(sources, refs, getContext());
List<ScanOutputItem> res = new ArrayList<>(); List<ScanOutputItem> res = new ArrayList<>();
for (String ref : refs) { for (SourceFile ref : refs) {
Content cnt = getIgLoader().loadContent(ref, "validate", false); Content cnt = getIgLoader().loadContent(ref.getRef(), "validate", false);
List<ValidationMessage> messages = new ArrayList<>(); List<ValidationMessage> messages = new ArrayList<>();
Element e = null; Element e = null;
try { try {
System.out.println("Validate " + ref); System.out.println("Validate " + ref);
messages.clear(); messages.clear();
e = getValidator().validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType()); e = getValidator().validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType());
res.add(new ScanOutputItem(ref, null, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine()))); res.add(new ScanOutputItem(ref.getRef(), null, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
} catch (Exception ex) { } catch (Exception ex) {
res.add(new ScanOutputItem(ref, null, null, exceptionToOutcome(ex))); res.add(new ScanOutputItem(ref.getRef(), null, null, exceptionToOutcome(ex)));
} }
if (e != null) { if (e != null) {
String rt = e.fhirType(); String rt = e.fhirType();
@ -104,9 +105,9 @@ public class Scanner {
System.out.println("Validate " + ref + " against " + ig.getUrl()); System.out.println("Validate " + ref + " against " + ig.getUrl());
messages.clear(); messages.clear();
getValidator().validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType(), url); getValidator().validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType(), url);
res.add(new ScanOutputItem(ref, ig, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine()))); res.add(new ScanOutputItem(ref.getRef(), ig, null, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
} catch (Exception ex) { } catch (Exception ex) {
res.add(new ScanOutputItem(ref, ig, null, exceptionToOutcome(ex))); res.add(new ScanOutputItem(ref.getRef(), ig, null, exceptionToOutcome(ex)));
} }
} }
Set<String> done = new HashSet<>(); Set<String> done = new HashSet<>();
@ -118,9 +119,9 @@ public class Scanner {
System.out.println("Validate " + ref + " against " + sd.getUrl()); System.out.println("Validate " + ref + " against " + sd.getUrl());
messages.clear(); messages.clear();
validator.validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType(), Collections.singletonList(sd)); validator.validate(null, messages, new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType(), Collections.singletonList(sd));
res.add(new ScanOutputItem(ref, ig, sd, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine()))); res.add(new ScanOutputItem(ref.getRef(), ig, sd, ValidatorUtils.messagesToOutcome(messages, getContext(), getFhirPathEngine())));
} catch (Exception ex) { } catch (Exception ex) {
res.add(new ScanOutputItem(ref, ig, sd, exceptionToOutcome(ex))); res.add(new ScanOutputItem(ref.getRef(), ig, sd, exceptionToOutcome(ex)));
} }
} }
} }

94
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java
View File

@ -88,6 +88,8 @@ import org.hl7.fhir.utilities.settings.FhirSettings;
import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.utilities.xhtml.XhtmlComposer; import org.hl7.fhir.utilities.xhtml.XhtmlComposer;
import org.hl7.fhir.validation.BaseValidator.ValidationControl; import org.hl7.fhir.validation.BaseValidator.ValidationControl;
import org.hl7.fhir.validation.ValidationEngine.IValidationEngineLoader;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.model.HtmlInMarkdownCheck; import org.hl7.fhir.validation.cli.model.HtmlInMarkdownCheck;
import org.hl7.fhir.validation.cli.services.IPackageInstaller; import org.hl7.fhir.validation.cli.services.IPackageInstaller;
import org.hl7.fhir.validation.cli.utils.ProfileLoader; import org.hl7.fhir.validation.cli.utils.ProfileLoader;
@ -176,6 +178,13 @@ POSSIBILITY OF SUCH DAMAGE.
@Accessors(chain = true) @Accessors(chain = true)
public class ValidationEngine implements IValidatorResourceFetcher, IValidationPolicyAdvisor, IPackageInstaller, IWorkerContextManager.IPackageLoadingTracker { public class ValidationEngine implements IValidatorResourceFetcher, IValidationPolicyAdvisor, IPackageInstaller, IWorkerContextManager.IPackageLoadingTracker {
public interface IValidationEngineLoader {
void load(Content cnt) throws FHIRException, IOException;
}
@Getter @Setter private SimpleWorkerContext context; @Getter @Setter private SimpleWorkerContext context;
@Getter @Setter private Map<String, byte[]> binaries = new HashMap<>(); @Getter @Setter private Map<String, byte[]> binaries = new HashMap<>();
@Getter @Setter private boolean doNative; @Getter @Setter private boolean doNative;
@ -423,7 +432,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
if (userAgent != null) { if (userAgent != null) {
contextBuilder.withUserAgent(userAgent); contextBuilder.withUserAgent(userAgent);
} }
context = contextBuilder.fromPackage(npm, ValidatorUtils.loaderForVersion(version)); context = contextBuilder.fromPackage(npm, ValidatorUtils.loaderForVersion(version), false);
} else { } else {
Map<String, byte[]> source = igLoader.loadIgSource(src, recursive, true); Map<String, byte[]> source = igLoader.loadIgSource(src, recursive, true);
if (version == null) { if (version == null) {
@ -543,35 +552,63 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
return list; return list;
} }
public OperationOutcome validate(String source, List<String> profiles) throws FHIRException, IOException { public OperationOutcome validate(String source, List<String> profiles, IValidationEngineLoader loader, boolean all) throws FHIRException, IOException {
List<String> l = new ArrayList<String>(); List<String> l = new ArrayList<String>();
List<SourceFile> refs = new ArrayList<>();
l.add(source); l.add(source);
return (OperationOutcome) validate(l, profiles, null); return (OperationOutcome) validate(l, profiles, refs, null, loader, all);
} }
public Resource validate(List<String> sources, List<String> profiles, List<ValidationRecord> record) throws FHIRException, IOException { public Resource validate(List<String> sources, List<String> profiles, List<SourceFile> refs, List<ValidationRecord> record, IValidationEngineLoader loader, boolean all) throws FHIRException, IOException {
if (profiles.size() > 0) {
System.out.println(" Profiles: " + profiles);
}
List<String> refs = new ArrayList<String>();
boolean asBundle = ValidatorUtils.parseSources(sources, refs, context); boolean asBundle = ValidatorUtils.parseSources(sources, refs, context);
Bundle results = new Bundle(); Bundle results = new Bundle();
results.setType(Bundle.BundleType.COLLECTION); results.setType(Bundle.BundleType.COLLECTION);
for (String ref : refs) { boolean found = false;
TimeTracker.Session tts = context.clock().start("validation");
context.clock().milestone(); for (SourceFile ref : refs) {
System.out.println(" Validate " + ref); if (ref.isProcess()) {
Content cnt = igLoader.loadContent(ref, "validate", false); found = true;
try { }
OperationOutcome outcome = validate(ref, cnt.getFocus(), cnt.getCntType(), profiles, record); }
ToolingExtensions.addStringExtension(outcome, ToolingExtensions.EXT_OO_FILE, ref); if (!found) {
System.out.println(" " + context.clock().milestone()); return null;
results.addEntry().setResource(outcome); }
tts.end();
} catch (Exception e) { // round one: try to read them all natively
System.out.println("Validation Infrastructure fail validating " + ref + ": " + e.getMessage()); // Ignore if it fails.The purpose of this is to make dependencies
tts.end(); // available for other resources to depend on. if it fails to load, there'll be an error if there's
throw new FHIRException(e); // something that should've been loaded
for (SourceFile ref : refs) {
if (ref.isProcess() || all) {
ref.setCnt(igLoader.loadContent(ref.getRef(), "validate", false));
if (loader != null) {
try {
loader.load(ref.getCnt());
} catch (Throwable t) {
System.out.println(t.getMessage());
}
}
}
}
for (SourceFile ref : refs) {
if (ref.isProcess() || all) {
TimeTracker.Session tts = context.clock().start("validation");
context.clock().milestone();
System.out.println(" Validate " + ref);
try {
OperationOutcome outcome = validate(ref.getRef(), ref.getCnt().getFocus(), ref.getCnt().getCntType(), profiles, record);
ToolingExtensions.addStringExtension(outcome, ToolingExtensions.EXT_OO_FILE, ref.getRef());
System.out.println(" " + context.clock().milestone());
results.addEntry().setResource(outcome);
tts.end();
} catch (Exception e) {
System.out.println("Validation Infrastructure fail validating " + ref + ": " + e.getMessage());
tts.end();
throw new FHIRException(e);
}
ref.setProcess(false);
} }
} }
if (asBundle) if (asBundle)
@ -598,7 +635,16 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
System.out.println(location + ": " + validator.reportTimes()); System.out.println(location + ": " + validator.reportTimes());
} }
if (record != null) { if (record != null) {
record.add(new ValidationRecord(location, messages)); boolean found = false;
for (ValidationRecord t : record) {
if (t.getLocation().equals(location)) {
found = true;
t.setMessages(messages);
}
}
if (!found) {
record.add(new ValidationRecord(location, messages));
}
} }
return ValidatorUtils.messagesToOutcome(messages, context, fhirPathEngine); return ValidatorUtils.messagesToOutcome(messages, context, fhirPathEngine);
} }

5
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationRecord.java
View File

@ -46,4 +46,9 @@ public class ValidationRecord {
return info; return info;
} }
public void setMessages(List<ValidationMessage> messages) {
this.messages = messages;
}
} }

2
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java
View File

@ -317,8 +317,6 @@ public class ValidatorCli {
} }
System.out.println("Loading"); System.out.println("Loading");
// Comment this out because definitions filename doesn't necessarily contain version (and many not even be 14 characters long).
// Version gets spit out a couple of lines later after we've loaded the context
String definitions = "dev".equals(cliContext.getSv()) ? "hl7.fhir.r5.core#current" : VersionUtilities.packageForVersion(cliContext.getSv()) + "#" + VersionUtilities.getCurrentVersion(cliContext.getSv()); String definitions = "dev".equals(cliContext.getSv()) ? "hl7.fhir.r5.core#current" : VersionUtilities.packageForVersion(cliContext.getSv()) + "#" + VersionUtilities.getCurrentVersion(cliContext.getSv());
ValidationEngine validator = validationService.initializeValidator(cliContext, definitions, tt); ValidationEngine validator = validationService.initializeValidator(cliContext, definitions, tt);
tts.end(); tts.end();

68
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorUtils.java
View File

@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -43,6 +44,33 @@ import org.xml.sax.SAXException;
//TODO find a home for these and clean it up //TODO find a home for these and clean it up
public class ValidatorUtils { public class ValidatorUtils {
public static class SourceFile {
private String ref;
private long date;
private boolean process;
private Content cnt;
public boolean isProcess() {
return process;
}
public void setProcess(boolean process) {
this.process = process;
this.cnt = null;
}
public String getRef() {
return ref;
}
public long getDate() {
return date;
}
public Content getCnt() {
return cnt;
}
public void setCnt(Content cnt) {
this.cnt = cnt;
}
}
protected static void grabNatives(Map<String, byte[]> source, Map<String, byte[]> binaries, String prefix) { protected static void grabNatives(Map<String, byte[]> source, Map<String, byte[]> binaries, String prefix) {
for (Map.Entry<String, byte[]> e : source.entrySet()) { for (Map.Entry<String, byte[]> e : source.entrySet()) {
if (e.getKey().endsWith(".zip")) if (e.getKey().endsWith(".zip"))
@ -129,14 +157,15 @@ public class ValidatorUtils {
* *
* @return {@link Boolean#TRUE} if more than one reference is found. * @return {@link Boolean#TRUE} if more than one reference is found.
*/ */
static boolean extractReferences(String name, List<String> refs, SimpleWorkerContext context) throws IOException { static boolean extractReferences(String name, List<SourceFile> refs, SimpleWorkerContext context) throws IOException {
if (Common.isNetworkPath(name)) { if (Common.isNetworkPath(name)) {
refs.add(name); SourceFile src = addSourceFile(refs, name);
src.date = Long.MAX_VALUE;
} else if (Common.isWildcardPath(name)) { } else if (Common.isWildcardPath(name)) {
AsteriskFilter filter = new AsteriskFilter(name); AsteriskFilter filter = new AsteriskFilter(name);
File[] files = new File(filter.getDir()).listFiles(filter); File[] files = new File(filter.getDir()).listFiles(filter);
for (File file : files) { for (File file : files) {
refs.add(file.getPath()); addSourceFile(refs, file);
} }
} else { } else {
File file = new File(name); File file = new File(name);
@ -150,24 +179,49 @@ public class ValidatorUtils {
} }
if (file.isFile()) { if (file.isFile()) {
refs.add(name); addSourceFile(refs, file);
} else { } else {
for (int i = 0; i < file.listFiles().length; i++) { for (int i = 0; i < file.listFiles().length; i++) {
File[] fileList = file.listFiles(); File[] fileList = file.listFiles();
if (fileList[i].isFile()) if (fileList[i].isFile()) {
refs.add(fileList[i].getPath()); if (!Utilities.isIgnorableFile(fileList[i])) {
addSourceFile(refs, fileList[i]);
}
}
} }
} }
} }
return refs.size() > 1; return refs.size() > 1;
} }
private static SourceFile addSourceFile(List<SourceFile> refs, File file) {
SourceFile src = addSourceFile(refs, file.getPath());
Long l = file.lastModified();
if (src.date != l) {
src.process = true;
}
src.date = l;
return src;
}
private static SourceFile addSourceFile(List<SourceFile> refs, String path) {
for (SourceFile t : refs) {
if (t.ref.equals(path)) {
return t;
}
}
SourceFile src = new SourceFile();
src.ref = path;
refs.add(src);
return src;
}
/** /**
* Iterates through the list of passed in sources, extracting all references and populated them in the passed in list. * Iterates through the list of passed in sources, extracting all references and populated them in the passed in list.
* *
* @return {@link Boolean#TRUE} if more than one reference is found. * @return {@link Boolean#TRUE} if more than one reference is found.
*/ */
public static boolean parseSources(List<String> sources, List<String> refs, SimpleWorkerContext context) throws IOException { public static boolean parseSources(List<String> sources, List<SourceFile> refs, SimpleWorkerContext context) throws IOException {
boolean multipleRefsFound = sources.size() > 1; boolean multipleRefsFound = sources.size() > 1;
for (String source : sources) { for (String source : sources) {
multipleRefsFound |= extractReferences(source, refs, context); multipleRefsFound |= extractReferences(source, refs, context);

144
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidationService.java
View File

@ -65,8 +65,10 @@ import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.validation.Content; import org.hl7.fhir.validation.Content;
import org.hl7.fhir.validation.IgLoader; import org.hl7.fhir.validation.IgLoader;
import org.hl7.fhir.validation.ValidationEngine; import org.hl7.fhir.validation.ValidationEngine;
import org.hl7.fhir.validation.ValidationEngine.IValidationEngineLoader;
import org.hl7.fhir.validation.ValidationRecord; import org.hl7.fhir.validation.ValidationRecord;
import org.hl7.fhir.validation.ValidatorUtils; import org.hl7.fhir.validation.ValidatorUtils;
import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
import org.hl7.fhir.validation.cli.model.CliContext; import org.hl7.fhir.validation.cli.model.CliContext;
import org.hl7.fhir.validation.cli.model.FileInfo; import org.hl7.fhir.validation.cli.model.FileInfo;
import org.hl7.fhir.validation.cli.model.ValidationOutcome; import org.hl7.fhir.validation.cli.model.ValidationOutcome;
@ -142,66 +144,86 @@ public class ValidationService {
} }
public void validateSources(CliContext cliContext, ValidationEngine validator) throws Exception { public void validateSources(CliContext cliContext, ValidationEngine validator) throws Exception {
long start = System.currentTimeMillis(); if (cliContext.getProfiles().size() > 0) {
List<ValidationRecord> records = new ArrayList<>(); System.out.println(" Profiles: " + cliContext.getProfiles());
Resource r = validator.validate(cliContext.getSources(), cliContext.getProfiles(), records);
MemoryMXBean mbean = ManagementFactory.getMemoryMXBean();
System.out.println("Done. " + validator.getContext().clock().report()+". Memory = "+Utilities.describeSize(mbean.getHeapMemoryUsage().getUsed()+mbean.getNonHeapMemoryUsage().getUsed()));
System.out.println();
PrintStream dst = null;
ValidationOutputRenderer renderer = makeValidationOutputRenderer(cliContext);
renderer.setCrumbTrails(validator.isCrumbTrails());
renderer.setRunDate(runDate);
if (renderer.isSingleFile()) {
if (cliContext.getOutput() == null) {
dst = System.out;
} else {
dst = new PrintStream(new FileOutputStream(cliContext.getOutput()));
}
renderer.setOutput(dst);
} else {
File dir = new File(cliContext.getOutput());
if (!dir.isDirectory()) {
throw new Error("The output location "+dir.getAbsolutePath()+" must be an existing directory for the output style "+renderer.getStyleCode());
}
renderer.setFolder(dir);
} }
ValidatorWatchMode watch = ValidatorWatchMode.NONE;
IgLoader igLoader = new IgLoader(validator.getPcm(), validator.getContext(), validator.getVersion());
List<ValidationRecord> records = new ArrayList<>();
List<SourceFile> refs = new ArrayList<>();
int ec = 0; int ec = 0;
if (r instanceof Bundle) { do {
if (renderer.handlesBundleDirectly()) { long start = System.currentTimeMillis();
renderer.render((Bundle) r); Resource r = validator.validate(cliContext.getSources(), cliContext.getProfiles(), refs, records, igLoader, watch == ValidatorWatchMode.ALL);
} else { boolean statusNeeded = false;
renderer.start(((Bundle) r).getEntry().size() > 1); if (r != null) {
for (Bundle.BundleEntryComponent e : ((Bundle) r).getEntry()) { statusNeeded = true;
OperationOutcome op = (OperationOutcome) e.getResource(); MemoryMXBean mbean = ManagementFactory.getMemoryMXBean();
ec = ec + countErrors(op); System.out.println("Done. " + validator.getContext().clock().report()+". Memory = "+Utilities.describeSize(mbean.getHeapMemoryUsage().getUsed()+mbean.getNonHeapMemoryUsage().getUsed()));
renderer.render(op); System.out.println();
PrintStream dst = null;
ValidationOutputRenderer renderer = makeValidationOutputRenderer(cliContext);
renderer.setCrumbTrails(validator.isCrumbTrails());
renderer.setRunDate(runDate);
if (renderer.isSingleFile()) {
if (cliContext.getOutput() == null) {
dst = System.out;
} else {
dst = new PrintStream(new FileOutputStream(cliContext.getOutput()));
}
renderer.setOutput(dst);
} else {
File dir = new File(cliContext.getOutput());
if (!dir.isDirectory()) {
throw new Error("The output location "+dir.getAbsolutePath()+" must be an existing directory for the output style "+renderer.getStyleCode());
}
renderer.setFolder(dir);
}
if (r instanceof Bundle) {
if (renderer.handlesBundleDirectly()) {
renderer.render((Bundle) r);
} else {
renderer.start(((Bundle) r).getEntry().size() > 1);
for (Bundle.BundleEntryComponent e : ((Bundle) r).getEntry()) {
OperationOutcome op = (OperationOutcome) e.getResource();
ec = ec + countErrors(op);
renderer.render(op);
}
renderer.finish();
}
} else if (r == null) {
ec = ec + 1;
System.out.println("No output from validation - nothing to validate");
} else {
renderer.start(false);
OperationOutcome op = (OperationOutcome) r;
ec = countErrors(op);
renderer.render((OperationOutcome) r);
renderer.finish();
}
if (cliContext.getOutput() != null && dst != null) {
dst.close();
}
if (cliContext.getHtmlOutput() != null) {
String html = new HTMLOutputGenerator(records).generate(System.currentTimeMillis() - start);
TextFile.stringToFile(html, cliContext.getHtmlOutput());
System.out.println("HTML Summary in " + cliContext.getHtmlOutput());
} }
renderer.finish();
} }
} else if (r == null) { if (watch != ValidatorWatchMode.NONE) {
ec = ec + 1; if (statusNeeded) {
System.out.println("No output from validation - nothing to validate"); System.out.println("Watching for changes");
} else { }
renderer.start(false); Thread.sleep(1000);
OperationOutcome op = (OperationOutcome) r; }
ec = countErrors(op); } while (watch != ValidatorWatchMode.NONE);
renderer.render((OperationOutcome) r);
renderer.finish();
}
if (cliContext.getOutput() != null && dst != null) {
dst.close();
}
if (cliContext.getHtmlOutput() != null) {
String html = new HTMLOutputGenerator(records).generate(System.currentTimeMillis() - start);
TextFile.stringToFile(html, cliContext.getHtmlOutput());
System.out.println("HTML Summary in " + cliContext.getHtmlOutput());
}
System.exit(ec > 0 ? 1 : 0); System.exit(ec > 0 ? 1 : 0);
} }
@ -552,11 +574,11 @@ public class ValidationService {
XLIFFProducer xliff = new XLIFFProducer(Utilities.path(dst)); XLIFFProducer xliff = new XLIFFProducer(Utilities.path(dst));
JsonLangFileProducer jl = new JsonLangFileProducer(Utilities.path(dst)); JsonLangFileProducer jl = new JsonLangFileProducer(Utilities.path(dst));
List<String> refs = new ArrayList<String>(); List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(cliContext.getSources(), refs, validator.getContext()); ValidatorUtils.parseSources(cliContext.getSources(), refs, validator.getContext());
for (String ref : refs) { for (SourceFile ref : refs) {
System.out.println(" Extract Translations from " + ref); System.out.println(" Extract Translations from " + ref);
org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref, "translate", false); org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref.getRef(), "translate", false);
Element e = Manager.parseSingle(validator.getContext(), new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType()); Element e = Manager.parseSingle(validator.getContext(), new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType());
LanguageProducerSession ps = po.startSession(e.fhirType()+"-"+e.getIdBase(), cliContext.getSrcLang()); LanguageProducerSession ps = po.startSession(e.fhirType()+"-"+e.getIdBase(), cliContext.getSrcLang());
LanguageProducerLanguageSession psl = ps.forLang(cliContext.getTgtLang()); LanguageProducerLanguageSession psl = ps.forLang(cliContext.getTgtLang());
@ -586,15 +608,15 @@ public class ValidationService {
loadTranslationSource(translations, input); loadTranslationSource(translations, input);
} }
List<String> refs = new ArrayList<String>(); List<SourceFile> refs = new ArrayList<>();
ValidatorUtils.parseSources(cliContext.getSources(), refs, validator.getContext()); ValidatorUtils.parseSources(cliContext.getSources(), refs, validator.getContext());
int t = 0; int t = 0;
for (String ref : refs) { for (SourceFile ref : refs) {
System.out.println(" Inject Translations into " + ref); System.out.println(" Inject Translations into " + ref);
org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref, "translate", false); org.hl7.fhir.validation.Content cnt = validator.getIgLoader().loadContent(ref.getRef(), "translate", false);
Element e = Manager.parseSingle(validator.getContext(), new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType()); Element e = Manager.parseSingle(validator.getContext(), new ByteArrayInputStream(cnt.getFocus()), cnt.getCntType());
t = t + new LanguageUtils(validator.getContext()).importFromTranslations(e, translations); t = t + new LanguageUtils(validator.getContext()).importFromTranslations(e, translations);
Manager.compose(validator.getContext(), e, new FileOutputStream(Utilities.path(dst, new File(ref).getName())), cnt.getCntType(), Manager.compose(validator.getContext(), e, new FileOutputStream(Utilities.path(dst, new File(ref.getRef()).getName())), cnt.getCntType(),
OutputStyle.PRETTY, null); OutputStyle.PRETTY, null);
} }
System.out.println("Done - imported "+t+" translations into "+refs.size()+ " in "+dst); System.out.println("Done - imported "+t+" translations into "+refs.size()+ " in "+dst);

8
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidatorWatchMode.java Normal file
View File

@ -0,0 +1,8 @@
package org.hl7.fhir.validation.cli.services;
public enum ValidatorWatchMode {
NONE, // just stop when validation is done
SINGLE, // when validation is done, watch the content that was validated, and revalidate anything that changes
ALL // when validation is done, watch the content that was validated, and revalidate everything if anything changes
}