From b2d2bebd85a49d01ce0a67ee2edcf71318092898 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Mon, 9 Sep 2019 13:32:11 +1000 Subject: [PATCH] Ad scan mode to validator --- .../fhir/r5/context/BaseWorkerContext.java | 7 + .../fhir/r5/context/SimpleWorkerContext.java | 1 + .../fhir/r5/validation/ValidationEngine.java | 401 +++++++++++++++++- .../org/hl7/fhir/r5/validation/Validator.java | 60 ++- 4 files changed, 447 insertions(+), 22 deletions(-) diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java index b4178c047..c2625ea2d 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java @@ -1201,4 +1201,11 @@ public abstract class BaseWorkerContext implements IWorkerContext { } + + public List allImplementationGuides() { + List res = new ArrayList<>(); + res.addAll(guides.values()); + return res; + } + } diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java index fd9fa4d31..64894d04a 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java @@ -53,6 +53,7 @@ import org.hl7.fhir.r5.formats.XmlParser; import org.hl7.fhir.r5.model.Bundle; import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent; import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r5.model.ImplementationGuide; import org.hl7.fhir.r5.model.MetadataResource; import org.hl7.fhir.r5.model.Questionnaire; import org.hl7.fhir.r5.model.Resource; diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/ValidationEngine.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/ValidationEngine.java index 038115f2e..ab2f5e489 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/ValidationEngine.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/ValidationEngine.java @@ -1,5 +1,7 @@ package org.hl7.fhir.r5.validation; +import java.io.BufferedOutputStream; + /*- * #%L * org.hl7.fhir.validation @@ -55,19 +57,28 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintWriter; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -94,6 +105,7 @@ import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.r5.conformance.ProfileUtilities; import org.hl7.fhir.r5.context.SimpleWorkerContext; import org.hl7.fhir.r5.context.SimpleWorkerContext.IContextResourceLoader; +import org.hl7.fhir.r5.elementmodel.Element; import org.hl7.fhir.r5.elementmodel.Manager; import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; import org.hl7.fhir.r5.formats.FormatUtilities; @@ -107,9 +119,12 @@ import org.hl7.fhir.r5.model.Constants; import org.hl7.fhir.r5.model.DomainResource; import org.hl7.fhir.r5.model.FhirPublication; import org.hl7.fhir.r5.model.ImplementationGuide; +import org.hl7.fhir.r5.model.ImplementationGuide.ImplementationGuideGlobalComponent; +import org.hl7.fhir.r5.model.ImplementationGuide.ManifestResourceComponent; import org.hl7.fhir.r5.model.OperationOutcome; import org.hl7.fhir.r5.model.OperationOutcome.OperationOutcomeIssueComponent; import org.hl7.fhir.r5.model.Parameters; +import org.hl7.fhir.r5.model.Reference; import org.hl7.fhir.r5.model.Resource; import org.hl7.fhir.r5.model.ResourceFactory; import org.hl7.fhir.r5.model.StructureDefinition; @@ -124,6 +139,7 @@ import org.hl7.fhir.r5.utils.NarrativeGenerator; import org.hl7.fhir.r5.utils.OperationOutcomeUtilities; import org.hl7.fhir.r5.utils.StructureMapUtilities; import org.hl7.fhir.r5.utils.StructureMapUtilities.ITransformerServices; +import org.hl7.fhir.r5.validation.ValidationEngine.ScanOutputItem; import org.hl7.fhir.r5.utils.ToolingExtensions; import org.hl7.fhir.r5.utils.ValidationProfileSet; import org.hl7.fhir.utilities.IniFile; @@ -136,6 +152,7 @@ import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType; import org.hl7.fhir.utilities.validation.ValidationMessage.Source; +import org.hl7.fhir.utilities.xhtml.XhtmlComposer; import org.xml.sax.SAXException; /** @@ -185,7 +202,47 @@ import org.xml.sax.SAXException; */ public class ValidationEngine { - public class TransformSupportServices implements ITransformerServices { + public class ScanOutputItem { + private String ref; + private ImplementationGuide ig; + private StructureDefinition profile; + private OperationOutcome outcome; + private String id; + public ScanOutputItem(String ref, ImplementationGuide ig, StructureDefinition profile, OperationOutcome outcome) { + super(); + this.ref = ref; + this.ig = ig; + this.profile = profile; + this.outcome = outcome; + } + public String getRef() { + return ref; + } + public ImplementationGuide getIg() { + return ig; + } + public StructureDefinition getProfile() { + return profile; + } + public OperationOutcome getOutcome() { + return outcome; + } + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public String getTitle() { + if (profile != null) + return "Validate " +ref+" against "+profile.present()+" ("+profile.getUrl()+")"; + if (ig != null) + return "Validate " +ref+" against global profile specified in "+ig.present()+" ("+ig.getUrl()+")"; + return "Validate " +ref+" against FHIR Spec"; + } + } + + public class TransformSupportServices implements ITransformerServices { private List outputs; @@ -476,6 +533,7 @@ public class ValidationEngine { InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(src, "package.tgz"), true); if (stream != null) return loadPackage(stream, Utilities.pathURL(src, "package.tgz")); + // todo: these options are deprecated - remove once all IGs have been rebuilt post R4 technical correction stream = fetchFromUrlSpecific(Utilities.pathURL(src, "igpack.zip"), true); if (stream != null) return readZip(stream); @@ -483,12 +541,18 @@ public class ValidationEngine { if (stream != null) return readZip(stream); stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true); + //// ----- + + // ok, having tried all that... now we'll just try to access it directly + if (stream == null) + stream = fetchFromUrlSpecific(src, "application/json", true); + FhirFormat fmt = checkIsResource(stream, src); if (fmt != null) { Map res = new HashMap(); res.put(Utilities.changeFileExt(src, "."+fmt.getExtension()), TextFile.fileToBytes(src)); return res; - } + } throw new Exception("Unable to find/resolve/read -ig "+src); } @@ -505,6 +569,20 @@ public class ValidationEngine { } } + private InputStream fetchFromUrlSpecific(String source, String contentType, boolean optional) throws Exception { + try { + URL url = new URL(source+"?nocache=" + System.currentTimeMillis()); + URLConnection c = url.openConnection(); + c.setRequestProperty("Content-Type", contentType); + return c.getInputStream(); + } catch (Exception e) { + if (optional) + return null; + else + throw e; + } + } + private Map scanDirectory(File f, boolean recursive) throws FileNotFoundException, IOException { Map res = new HashMap<>(); for (File ff : f.listFiles()) { @@ -835,6 +913,76 @@ public class ValidationEngine { return (OperationOutcome)validate(l, profiles); } + public List validateScan(List sources, Set guides) throws Exception { + List refs = new ArrayList(); + handleSources(sources, refs); + + List res = new ArrayList(); + InstanceValidator validator = getValidator(); + + for (String ref : refs) { + Content cnt = loadContent(ref, "validate"); + List messages = new ArrayList(); + Element e = null; + try { + System.out.println("Validate "+ref); + messages.clear(); + e = validator.validate(null, messages, new ByteArrayInputStream(cnt.focus), cnt.cntType); + res.add(new ScanOutputItem(ref, null, null, messagesToOutcome(messages))); + } catch (Exception ex) { + res.add(new ScanOutputItem(ref, null, null, exceptionToOutcome(ex))); + } + if (e != null) { + String rt = e.fhirType(); + for (String u : guides) { + ImplementationGuide ig = context.fetchResource(ImplementationGuide.class, u); + System.out.println("Check Guide "+ig.getUrl()); + String canonical = ig.getUrl().contains("/Impl") ? ig.getUrl().substring(0, ig.getUrl().indexOf("/Impl")) : ig.getUrl(); + String url = getGlobal(ig, rt); + if (url != null) { + try { + System.out.println("Validate "+ref+" against "+ig.getUrl()); + messages.clear(); + validator.validate(null, messages, new ByteArrayInputStream(cnt.focus), cnt.cntType, new ValidationProfileSet(url, true)); + res.add(new ScanOutputItem(ref, ig, null, messagesToOutcome(messages))); + } catch (Exception ex) { + res.add(new ScanOutputItem(ref, ig, null, exceptionToOutcome(ex))); + } + } + Set done = new HashSet<>(); + for (StructureDefinition sd : context.allStructures()) { + if (!done.contains(sd.getUrl())) { + done.add(sd.getUrl()); + if (sd.getUrl().startsWith(canonical) && rt.equals(sd.getType())) { + try { + System.out.println("Validate "+ref+" against "+sd.getUrl()); + messages.clear(); + validator.validate(null, messages, new ByteArrayInputStream(cnt.focus), cnt.cntType, new ValidationProfileSet(sd.getUrl(), true)); + res.add(new ScanOutputItem(ref, ig, sd, messagesToOutcome(messages))); + } catch (Exception ex) { + res.add(new ScanOutputItem(ref, ig, sd, exceptionToOutcome(ex))); + } + } + } + } + } + } + } + return res; + } + + private Resource resolve(Reference reference) { + return null; + } + + private String getGlobal(ImplementationGuide ig, String rt) { + for (ImplementationGuideGlobalComponent igg : ig.getGlobal()) { + if (rt.equals(igg.getType())) + return igg.getProfile(); + } + return null; + } + public Resource validate(List sources, List profiles) throws Exception { List refs = new ArrayList(); boolean asBundle = handleSources(sources, refs); @@ -997,6 +1145,13 @@ public class ValidationEngine { return filteredValidation; } + private OperationOutcome exceptionToOutcome(Exception ex) throws DefinitionException { + OperationOutcome op = new OperationOutcome(); + op.addIssue().setCode(org.hl7.fhir.r5.model.OperationOutcome.IssueType.EXCEPTION).setSeverity(org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.FATAL).getDetails().setText(ex.getMessage()); + new NarrativeGenerator("", "", context).generate(null, op); + return op; + } + private OperationOutcome messagesToOutcome(List messages) throws DefinitionException { OperationOutcome op = new OperationOutcome(); for (ValidationMessage vm : filterMessages(messages)) { @@ -1136,5 +1291,245 @@ public class ValidationEngine { public void setDebug(boolean debug) { this.debug = debug; } - + + public void genScanOutput(String folder, List items) throws IOException { + String f = Utilities.path(folder, "comparison.zip"); + download("http://fhir.org/archive/comparison.zip", f); + unzip(f, folder); + + for (int i = 0; i < items.size(); i++) { + items.get(i).setId("c"+Integer.toString(i)); + genScanOutputItem(items.get(i), Utilities.path(folder, items.get(i).getId()+".html")); + } + + StringBuilder b = new StringBuilder(); + b.append(""); + b.append(""); + b.append("Implementation Guide Scan"); + b.append("\r\n"); + b.append("\r\n"); + b.append(""); + b.append(""); + b.append("

Implementation Guide Scan

"); + + // organise + Set refs = new HashSet<>(); + Set igs = new HashSet<>(); + Map> profiles = new HashMap<>(); + for (ScanOutputItem item : items) { + refs.add(item.ref); + if (item.ig != null) { + igs.add(item.ig.getUrl()); + if (!profiles.containsKey(item.ig.getUrl())) { + profiles.put(item.ig.getUrl(), new HashSet<>()); + } + if (item.profile != null) + profiles.get(item.ig.getUrl()).add(item.profile.getUrl()); + } + } + + b.append("

By reference

\r\n"); + b.append(""); + b.append(""); + for (String s : sorted(igs)) { + ImplementationGuide ig = context.fetchResource(ImplementationGuide.class, s); + b.append(""); + } + b.append("\r\n"); + b.append(""); + for (String s : sorted(igs)) { + ImplementationGuide ig = context.fetchResource(ImplementationGuide.class, s); + b.append(""); + for (String sp : sorted(profiles.get(s))) { + StructureDefinition sd = context.fetchResource(StructureDefinition.class, sp); + b.append(""); + } + } + b.append("\r\n"); + + for (String s : sorted(refs)) { + b.append(""); + b.append(""); + b.append(genOutcome(items, s, null, null)); + for (String si : sorted(igs)) { + ImplementationGuide ig = context.fetchResource(ImplementationGuide.class, si); + b.append(genOutcome(items, s, si, null)); + for (String sp : sorted(profiles.get(ig.getUrl()))) { + b.append(genOutcome(items, s, si, sp)); + } + } + b.append("\r\n"); + } + b.append("
"+ig.present()+"
SourceCore SpecGlobal"+sd.present()+"
"+s+"
\r\n"); + + b.append("

By IG

\r\n"); + b.append(""); + b.append(""); + for (String s : sorted(refs)) { + b.append(""); + } + b.append("\r\n"); + b.append(""); + for (String s : sorted(refs)) { + b.append(genOutcome(items, s, null, null)); + } + b.append("\r\n"); + for (String si : sorted(igs)) { + b.append(""); + ImplementationGuide ig = context.fetchResource(ImplementationGuide.class, si); + b.append(""); + b.append(""); + for (String s : sorted(refs)) { + b.append(genOutcome(items, s, si, null)); + } + b.append("\r\n"); + + for (String sp : sorted(profiles.get(ig.getUrl()))) { + b.append(""); + StructureDefinition sd = context.fetchResource(StructureDefinition.class, sp); + b.append(""); + for (String s : sorted(refs)) { + b.append(genOutcome(items, s, si, sp)); + } + b.append("\r\n"); + } + } + b.append("
"+s+"
Core Spec
"+ig.present()+"Global
"+sd.present()+"
\r\n"); + + b.append(""); + b.append(""); + TextFile.stringToFile(b.toString(), Utilities.path(folder, "scan.html")); + + + } + + private String genOutcome(List items, String src, String ig, String profile) { + ScanOutputItem item = null; + for (ScanOutputItem t : items) { + boolean match = true; + if (!t.ref.equals(src)) + match = false; + if (!((ig == null && t.ig == null) || (ig != null && t.ig != null && ig.equals(t.ig.getUrl())))) + match = false; + if (!((profile == null && t.profile == null) || (profile != null && t.profile != null && profile.equals(t.profile.getUrl())))) + match = false; + if (match) { + item = t; + break; + } + } + + if (item == null) + return ""; + boolean ok = true; + for (OperationOutcomeIssueComponent iss : item.outcome.getIssue()) { + if (iss.getSeverity() == org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.ERROR || iss.getSeverity() == org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.FATAL) { + ok = false; + } + } + if (ok) + return "\u2714"; + else + return "\u2716"; + } + + public void unzip(String zipFilePath, String destDirectory) throws IOException { + File destDir = new File(destDirectory); + if (!destDir.exists()) { + destDir.mkdir(); + } + ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath)); + ZipEntry entry = zipIn.getNextEntry(); + // iterates over entries in the zip file + while (entry != null) { + String filePath = destDirectory + File.separator + entry.getName(); + if (!entry.isDirectory()) { + // if the entry is a file, extracts it + extractFile(zipIn, filePath); + } else { + // if the entry is a directory, make the directory + File dir = new File(filePath); + dir.mkdir(); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + zipIn.close(); + } + + private static final int BUFFER_SIZE = 4096; + + private void extractFile(ZipInputStream zipIn, String filePath) throws IOException { + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath)); + byte[] bytesIn = new byte[BUFFER_SIZE]; + int read = 0; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + bos.close(); } + + private void download(String address, String filename) throws IOException { + URL url = new URL(address); + URLConnection c = url.openConnection(); + InputStream s = c.getInputStream(); + FileOutputStream f = new FileOutputStream(filename); + transfer(s, f, 1024); + f.close(); + } + + public static void transfer(InputStream in, OutputStream out, int buffer) throws IOException { + byte[] read = new byte[buffer]; // Your buffer size. + while (0 < (buffer = in.read(read))) + out.write(read, 0, buffer); + } + + private void genScanOutputItem(ScanOutputItem item, String filename) throws IOException { + NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context); + gen.setNoSlowLookup(true); + gen.generate(null, item.outcome); + String s = new XhtmlComposer(XhtmlComposer.HTML).compose(item.outcome.getText().getDiv()); + + String title = item.getTitle(); + + StringBuilder b = new StringBuilder(); + b.append(""); + b.append(""); + b.append(""+title+""); + b.append("\r\n"); + b.append(""); + b.append(""); + b.append("

"+title+"

"); + b.append(s); + b.append(""); + b.append(""); + TextFile.stringToFile(b.toString(), filename); + + } + + + private List sorted(Set keys) { + List names = new ArrayList(); + if (keys != null) + names.addAll(keys); + Collections.sort(names); + return names; + } + +} + diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/Validator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/Validator.java index cf721de02..56a562b83 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/Validator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/r5/validation/Validator.java @@ -58,8 +58,10 @@ import java.net.HttpURLConnection; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.hl7.fhir.r4.model.CapabilityStatement.CapabilityStatementKind; import org.hl7.fhir.r5.conformance.CapabilityStatementUtilities; @@ -81,7 +83,9 @@ import org.hl7.fhir.r5.model.OperationOutcome; import org.hl7.fhir.r5.model.OperationOutcome.OperationOutcomeIssueComponent; import org.hl7.fhir.r5.model.Resource; import org.hl7.fhir.r5.model.StructureDefinition; +import org.hl7.fhir.r5.utils.NarrativeGenerator; import org.hl7.fhir.r5.utils.ToolingExtensions; +import org.hl7.fhir.r5.validation.ValidationEngine.ScanOutputItem; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.VersionUtil; @@ -106,7 +110,7 @@ import org.hl7.fhir.utilities.xhtml.XhtmlComposer; public class Validator { public enum EngineMode { - VALIDATION, TRANSFORM, NARRATIVE, SNAPSHOT + VALIDATION, TRANSFORM, NARRATIVE, SNAPSHOT, SCAN } private static String getNamedParam(String[] args, String param) { @@ -339,7 +343,7 @@ public class Validator { String lang = null; boolean doDebug = false; - // load the parameters - so order doesn't matter + // load the parameters - so order doesn't matter for (int i = 0; i < args.length; i++) { if (args[i].equals("-defn")) if (i+1 == args.length) @@ -396,6 +400,8 @@ public class Validator { mode = EngineMode.NARRATIVE; else if (args[i].equals("-snapshot")) mode = EngineMode.SNAPSHOT; + else if (args[i].equals("-scan")) + mode = EngineMode.SCAN; else if (args[i].equals("-tx")) if (i+1 == args.length) throw new Error("Specified -tx without indicating terminology server"); @@ -510,25 +516,41 @@ public class Validator { validator.loadProfile(locations.getOrDefault(s, s)); } } - if (profiles.size() > 0) - System.out.println(" .. validate "+sources+" against "+profiles.toString()); - else - System.out.println(" .. validate "+sources); - validator.prepare(); // generate any missing snapshots - Resource r = validator.validate(sources, profiles); - int ec = 0; - if (output == null) { - if (r instanceof Bundle) - for (BundleEntryComponent e : ((Bundle)r).getEntry()) - ec = displayOO((OperationOutcome)e.getResource()) + ec; + if (mode == EngineMode.SCAN) { + if (Utilities.noString(output)) + throw new Exception("Output parameter required when scanning"); + if (!(new File(output).isDirectory())) + throw new Exception("Output '"+output+"' must be a directory when scanning"); + System.out.println(" .. scan "+sources+" against loaded IGs"); + Set urls = new HashSet<>(); + for (ImplementationGuide ig : validator.getContext().allImplementationGuides()) { + if (ig.getUrl().contains("/ImplementationGuide") && !ig.getUrl().equals("http://hl7.org/fhir/ImplementationGuide/fhir")) + urls.add(ig.getUrl()); + } + List res = validator.validateScan(sources, urls); + validator.genScanOutput(output, res); + System.out.println("Done. output in "+Utilities.path(output, "scan.html")); + } else { + if (profiles.size() > 0) + System.out.println(" .. validate "+sources+" against "+profiles.toString()); else - ec = displayOO((OperationOutcome)r); - } else { - FileOutputStream s = new FileOutputStream(output); - x.compose(s, r); - s.close(); + System.out.println(" .. validate "+sources); + validator.prepare(); // generate any missing snapshots + Resource r = validator.validate(sources, profiles); + int ec = 0; + if (output == null) { + if (r instanceof Bundle) + for (BundleEntryComponent e : ((Bundle)r).getEntry()) + ec = displayOO((OperationOutcome)e.getResource()) + ec; + else + ec = displayOO((OperationOutcome)r); + } else { + FileOutputStream s = new FileOutputStream(output); + x.compose(s, r); + s.close(); + } + System.exit(ec > 0 ? 1 : 0); } - System.exit(ec > 0 ? 1 : 0); } } }