From 47e205883cfde1c10b29ddae2c9885c32bdc7980 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Mon, 23 May 2022 12:23:23 +1000 Subject: [PATCH 1/2] Performance work in the validator --- .../fhir/r5/conformance/ProfileUtilities.java | 5 + .../fhir/r5/context/BaseWorkerContext.java | 31 +++ .../hl7/fhir/r5/context/IWorkerContext.java | 8 + .../fhir/r5/context/SimpleWorkerContext.java | 4 +- .../org/hl7/fhir/r5/elementmodel/Element.java | 152 ++++++++++--- .../hl7/fhir/utilities/SimpleTimeTracker.java | 90 ++++++++ .../org/hl7/fhir/utilities/Utilities.java | 8 + .../npm/BasePackageCacheManager.java | 1 + .../npm/FilesystemPackageCacheManager.java | 4 +- .../hl7/fhir/validation/BaseValidator.java | 63 ++++-- .../org/hl7/fhir/validation/IgLoader.java | 102 ++++++--- .../hl7/fhir/validation/ResourceChecker.java | 7 +- .../hl7/fhir/validation/ValidationEngine.java | 49 +++-- .../services/StandAloneValidatorFetcher.java | 4 + .../cli/services/ValidationService.java | 3 +- .../instance/InstanceValidator.java | 203 ++++++++++++------ .../instance/PercentageTracker.java | 38 ++++ .../instance/type/BundleValidator.java | 51 ++--- 18 files changed, 637 insertions(+), 186 deletions(-) create mode 100644 org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/SimpleTimeTracker.java create mode 100644 org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/ProfileUtilities.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/ProfileUtilities.java index 644c9be45..5b3852ba9 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/ProfileUtilities.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/ProfileUtilities.java @@ -341,6 +341,7 @@ public class ProfileUtilities extends TranslatingUtilities { private XVerExtensionManager xver; private boolean wantFixDifferentialFirstElementType; private Set masterSourceFileNames; + private Map> childMapCache = new HashMap<>(); public ProfileUtilities(IWorkerContext context, List messages, ProfileKnowledgeProvider pkp, FHIRPathEngine fpe) { super(); @@ -413,6 +414,9 @@ public class ProfileUtilities extends TranslatingUtilities { public List getChildMap(StructureDefinition profile, ElementDefinition element) throws DefinitionException { + if (childMapCache .containsKey(element)) { + return childMapCache.get(element); + } if (element.getContentReference() != null) { List list = null; String id = null; @@ -452,6 +456,7 @@ public class ProfileUtilities extends TranslatingUtilities { } else break; } + childMapCache.put(element, res); return res; } } diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java index 4cb741601..fd12bf050 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java @@ -54,6 +54,7 @@ import org.hl7.fhir.exceptions.NoTerminologyServiceException; import org.hl7.fhir.exceptions.TerminologyServiceException; import org.hl7.fhir.r5.conformance.ProfileUtilities; import org.hl7.fhir.r5.context.CanonicalResourceManager.CanonicalResourceProxy; +import org.hl7.fhir.r5.context.IWorkerContext.IPackageLoadingTracker; import org.hl7.fhir.r5.context.IWorkerContext.ILoggingService.LogCategory; import org.hl7.fhir.r5.context.TerminologyCache.CacheToken; import org.hl7.fhir.r5.model.BooleanType; @@ -216,6 +217,8 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte private final CanonicalResourceManager operations = new CanonicalResourceManager(false); private final CanonicalResourceManager plans = new CanonicalResourceManager(false); private final CanonicalResourceManager systems = new CanonicalResourceManager(false); + private Map systemUrlMap; + private UcumService ucumService; protected Map binaries = new HashMap(); @@ -280,6 +283,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte questionnaires.copy(other.questionnaires); operations.copy(other.operations); systems.copy(other.systems); + systemUrlMap = null; guides.copy(other.guides); capstmts.copy(other.capstmts); measures.copy(other.measures); @@ -443,11 +447,28 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte transforms.see((StructureMap) m, packageInfo); } else if (r instanceof NamingSystem) { systems.see((NamingSystem) m, packageInfo); + systemUrlMap = null; } } } } + public Map getNSUrlMap() { + if (systemUrlMap == null) { + systemUrlMap = new HashMap<>(); + List nsl = new ArrayList<>(); + for (NamingSystem ns : nsl) { + for (NamingSystemUniqueIdComponent uid : ns.getUniqueId()) { + if (uid.getType() == NamingSystemIdentifierType.URI && uid.hasValue()) { + systemUrlMap.put(uid.getValue(), ns) ; + } + } + } + } + return systemUrlMap; + } + + public void fixOldSD(StructureDefinition sd) { if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT && sd.getType().equals("Extension") && sd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition/")) { sd.setSnapshot(null); @@ -1661,6 +1682,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte private Set notCanonical = new HashSet(); private String overrideVersionNs; + protected IPackageLoadingTracker packageTracker; @Override public Resource fetchResourceById(String type, String uri) { @@ -1832,6 +1854,7 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte transforms.drop(id); } else if (fhirType.equals("NamingSystem")) { systems.drop(id); + systemUrlMap = null; } } } @@ -2269,4 +2292,12 @@ public abstract class BaseWorkerContext extends I18nBase implements IWorkerConte return false; } + public IPackageLoadingTracker getPackageTracker() { + return packageTracker; + } + + public IWorkerContext setPackageTracker(IPackageLoadingTracker packageTracker) { + this.packageTracker = packageTracker; + return this; + } } \ No newline at end of file diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/IWorkerContext.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/IWorkerContext.java index 95f9b3957..020ddc8b9 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/IWorkerContext.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/IWorkerContext.java @@ -57,6 +57,7 @@ import org.hl7.fhir.r5.model.CodeableConcept; import org.hl7.fhir.r5.model.Coding; import org.hl7.fhir.r5.model.ConceptMap; import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingComponent; +import org.hl7.fhir.r5.model.NamingSystem; import org.hl7.fhir.r5.model.Parameters; import org.hl7.fhir.r5.model.Resource; import org.hl7.fhir.r5.model.StructureDefinition; @@ -101,6 +102,10 @@ import javax.annotation.Nonnull; */ public interface IWorkerContext { + public interface IPackageLoadingTracker { + public void packageLoaded(String pid, String version); + } + public class CodingValidationRequest { private Coding coding; private ValidationResult result; @@ -790,6 +795,7 @@ public interface IWorkerContext { * @return */ public String oid2Uri(String code); + public Map getNSUrlMap(); /** * @return true if the contxt has a terminology caching service internally @@ -877,6 +883,8 @@ public interface IWorkerContext { public IWorkerContext setClientRetryCount(int value); public TimeTracker clock(); + public IPackageLoadingTracker getPackageTracker(); + public IWorkerContext setPackageTracker(IPackageLoadingTracker packageTracker); public PackageVersion getPackageForUrl(String url); } \ No newline at end of file diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java index 3b27e04fc..0ba485cec 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java @@ -469,7 +469,9 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon return 0; } loadedPackages.add(pi.id()+"#"+pi.version()); - + if (packageTracker != null) { + packageTracker.packageLoaded(pi.id(), pi.version()); + } if ((types == null || types.length == 0) && loader != null) { types = loader.getTypes(); diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Element.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Element.java index 86dc38ea3..f7d2ef3b7 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Element.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Element.java @@ -117,6 +117,9 @@ public class Element extends Base { private List messages; private boolean prohibited; private boolean required; + private Map> childMap; + private int descendentCount; + private int instanceId; public Element(String name) { super(); @@ -210,11 +213,19 @@ public class Element extends Base { public List getChildrenByName(String name) { List res = new ArrayList(); - if (hasChildren()) { - for (Element child : children) - if (name.equals(child.getName())) - res.add(child); - } + if (children.size() > 20) { + populateChildMap(); + List l = childMap.get(name); + if (l != null) { + res.addAll(l); + } + } else { + if (hasChildren()) { + for (Element child : children) + if (name.equals(child.getName())) + res.add(child); + } + } return res; } @@ -272,6 +283,7 @@ public class Element extends Base { child.setValue(value); } } + childMap = null; try { setProperty(name.hashCode(), name, new StringType(value)); } catch (FHIRException e) { @@ -279,13 +291,21 @@ public class Element extends Base { } } - public List getChildren(String name) { - List res = new ArrayList(); - if (children != null) - for (Element child : children) { - if (name.equals(child.getName())) - res.add(child); - } + public List getChildren(String name) { + List res = new ArrayList(); + if (children.size() > 20) { + populateChildMap(); + List l = childMap.get(name); + if (l != null) { + res.addAll(l); + } + } else { + if (children != null) + for (Element child : children) { + if (name.equals(child.getName())) + res.add(child); + } + } return res; } @@ -313,12 +333,22 @@ public class Element extends Base { List result = new ArrayList(); if (children != null) { - for (Element child : children) { - if (child.getName().equals(name)) - result.add(child); - if (child.getName().startsWith(name) && child.getProperty().isChoice() && child.getProperty().getName().equals(name+"[x]")) - result.add(child); - } + if (children.size() > 20) { + populateChildMap(); + List l = childMap.get(name); + if (l != null) { + result.addAll(l); + } + } else { + for (Element child : children) { + if (child.getName().equals(name)) { + result.add(child); + } + if (child.getName().startsWith(name) && child.getProperty().isChoice() && child.getProperty().getName().equals(name+"[x]")) { + result.add(child); + } + } + } } if (result.isEmpty() && checkValid) { // throw new FHIRException("not determined yet"); @@ -326,6 +356,24 @@ public class Element extends Base { return result.toArray(new Base[result.size()]); } + private void populateChildMap() { + if (childMap == null) { + childMap = new HashMap<>(); + for (Element child : children) { + String n = child.getName(); + if (n.endsWith("[x]")) { + n = n.substring(0, n.length()-3); + } + List l = childMap.get(n); + if (l == null) { + l = new ArrayList(); + childMap.put(n,l); + } + l.add(child); + } + } + } + @Override protected void listChildren(List childProps) { if (children != null) { @@ -362,6 +410,7 @@ public class Element extends Base { throw new FHIRException("Cannot set property "+name+" on "+this.name+" - value is not a primitive type ("+value.fhirType()+") or an ElementModel type"); } + childMap = null; if (children == null) children = new ArrayList(); Element childForValue = null; @@ -532,8 +581,10 @@ public class Element extends Base { public void clearDecorations() { clearUserData("fhir.decorations"); - for (Element e : children) + for (Element e : children) { e.clearDecorations(); + } + childMap = null; } public void markValidation(StructureDefinition profile, ElementDefinition definition) { @@ -557,7 +608,21 @@ public class Element extends Base { public Element getNamedChild(String name) { if (children == null) return null; + if (children.size() > 20) { + populateChildMap(); + List l = childMap.get(name); + if (l == null) { + // try the other way (in case of complicated naming rules) + } else if (l.size() > 1) { + throw new Error("Attempt to read a single element when there is more than one present ("+name+")"); + } else { + return l.get(0); + } + } else { + + } Element result = null; + for (Element child : children) { if (child.getName() != null && name != null && child.getProperty() != null && child.getProperty().getDefinition() != null && child.fhirType() != null) { if (child.getName().equals(name) || (child.getName().length() > child.fhirType().length() && child.getName().substring(0, child.getName().length() - child.fhirType().length()).equals(name) && child.getProperty().getDefinition().isChoice())) { @@ -573,9 +638,17 @@ public class Element extends Base { public void getNamedChildren(String name, List list) { if (children != null) - for (Element child : children) - if (child.getName().equals(name)) - list.add(child); + if (children.size() > 20) { + populateChildMap(); + List l = childMap.get(name); + if (l != null) { + list.addAll(l); + } + } else { + for (Element child : children) + if (child.getName().equals(name)) + list.add(child); + } } public String getNamedChildValue(String name) { @@ -761,6 +834,7 @@ public class Element extends Base { } children.removeAll(remove); Collections.sort(children, new ElementSortComparator(this, this.property)); + childMap = null; } } @@ -965,7 +1039,8 @@ public class Element extends Base { public void clear() { comments = null; - children.clear();; + children.clear(); + childMap = null; property = null; elementProperty = null; xhtml = null; @@ -996,7 +1071,8 @@ public class Element extends Base { } public void removeChild(String name) { - children.removeIf(n -> name.equals(n.getName())); + children.removeIf(n -> name.equals(n.getName())); + childMap = null; } public boolean isProhibited() { @@ -1014,6 +1090,34 @@ public class Element extends Base { public void setRequired(boolean required) { this.required = required; } + + public int getDescendentCount() { + return descendentCount; + } + + public void setDescendentCount(int descendentCount) { + this.descendentCount = descendentCount; + } + + public int countDescendents() { + if (descendentCount > 0) { + return descendentCount; + } else { + descendentCount = children.size(); + for (Element e : children) { + descendentCount = descendentCount + e.countDescendents(); + } + } + return descendentCount; + } + + public int getInstanceId() { + return instanceId; + } + + public void setInstanceId(int instanceId) { + this.instanceId = instanceId; + } } \ No newline at end of file diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/SimpleTimeTracker.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/SimpleTimeTracker.java new file mode 100644 index 000000000..c9e2905bf --- /dev/null +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/SimpleTimeTracker.java @@ -0,0 +1,90 @@ +package org.hl7.fhir.utilities; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; + +public class SimpleTimeTracker { + + private long start; + private String name; + private String context; + private int count; + + public static int level = 0; + public static BufferedWriter bw; + + public SimpleTimeTracker(String name) { + level++; + this.name = name; + count = -1; + start = System.currentTimeMillis(); + init(); + log(name); + } + + public SimpleTimeTracker(String name, String context) { + level++; + this.name = name; + this.context = context; + start = System.currentTimeMillis(); + count = -1; + init(); + log(name+" ["+context+"]"); + } + + private void init() { + if (bw == null) { + try { + File fout = new File("/Users/grahamegrieve/temp/time.txt"); + FileOutputStream fos = new FileOutputStream(fout); + bw = new BufferedWriter(new OutputStreamWriter(fos)); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + } + + public void report() { + String s = count == -1 ? "" : " ("+count+")"; + if (context == null) { + s = name+": "+(System.currentTimeMillis()-start)+"ms "+s; + } else { + s = name+": "+(System.currentTimeMillis()-start)+"ms "+s+" ["+context+"]"; + } + log(s); + level--; + if (level == 0) { + try { + bw.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + private void log(String s) { + System.out.println(Utilities.padLeft("", '#', level)+" "+s); + try { + bw.write(s); + bw.newLine(); + } catch (Exception e) { + System.out.println("e: " +e.getMessage()); + } + } + + public void count() { + if (count == -1) { + count = 1; + } else { + count++; + } + + } + +} diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java index 68bcf9eb5..96aa35479 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java @@ -1731,4 +1731,12 @@ public class Utilities { return splitTimezone(value)[0].replace("T", "").replace(":", "").replace(".", "").length(); } + public static String padInt(int i, int len) { + return Utilities.padLeft(Integer.toString(i), ' ', len); + } + + public static String padInt(long i, int len) { + return Utilities.padLeft(Long.toString(i), ' ', len); + } + } \ No newline at end of file diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java index d3834ebe9..22429928d 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.function.Function; public abstract class BasePackageCacheManager implements IPackageCacheManager { + private static final Logger ourLog = LoggerFactory.getLogger(BasePackageCacheManager.class); private List myPackageServers = new ArrayList<>(); private Function myClientFactory = address -> new CachingPackageClient(address); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java index 08e2a148a..b14fc3019 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java @@ -83,6 +83,8 @@ import java.util.Map.Entry; */ public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager { + + // private static final String SECONDARY_SERVER = "http://local.fhir.org:8080/packages"; public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$"; public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$"; @@ -96,7 +98,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple private Map ciList = new HashMap(); private JsonArray buildInfo; private boolean suppressErrors; - + /** * Constructor */ diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/BaseValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/BaseValidator.java index e6bffec2e..82463d169 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/BaseValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/BaseValidator.java @@ -802,15 +802,33 @@ public class BaseValidator implements IValidationContextResourceLoader { return null; } - protected Element resolveInBundle(List entries, String ref, String fullUrl, String type, String id) { - if (Utilities.isAbsoluteUrl(ref)) { - // if the reference is absolute, then you resolve by fullUrl. No other thinking is required. + protected Element resolveInBundle(Element bundle, List entries, String ref, String fullUrl, String type, String id) { + @SuppressWarnings("unchecked") + Map map = (Map) bundle.getUserData("validator.entrymap"); + if (map == null) { + map = new HashMap<>(); + bundle.setUserData("validator.entrymap", map); for (Element entry : entries) { String fu = entry.getNamedChildValue(FULL_URL); - if (ref.equals(fu)) - return entry; - } - return null; + map.put(fu, entry); + Element resource = entry.getNamedChild(RESOURCE); + if (resource != null) { + String et = resource.getType(); + String eid = resource.getNamedChildValue(ID); + map.put(et+"/"+eid, entry); + } + } + } + + if (Utilities.isAbsoluteUrl(ref)) { + // if the reference is absolute, then you resolve by fullUrl. No other thinking is required. + return map.get(ref); +// for (Element entry : entries) { +// String fu = entry.getNamedChildValue(FULL_URL); +// if (ref.equals(fu)) +// return entry; +// } +// return null; } else { // split into base, type, and id String u = null; @@ -822,20 +840,25 @@ public class BaseValidator implements IValidationContextResourceLoader { if (parts.length >= 2) { String t = parts[0]; String i = parts[1]; - for (Element entry : entries) { - String fu = entry.getNamedChildValue(FULL_URL); - if (fu != null && fu.equals(u)) - return entry; - if (u == null) { - Element resource = entry.getNamedChild(RESOURCE); - if (resource != null) { - String et = resource.getType(); - String eid = resource.getNamedChildValue(ID); - if (t.equals(et) && i.equals(eid)) - return entry; - } - } + Element res = map.get(u); + if (res == null) { + res = map.get(t+"/"+i); } + return res; +// for (Element entry : entries) { +// String fu = entry.getNamedChildValue(FULL_URL); +// if (fu != null && fu.equals(u)) +// return entry; +// if (u == null) { +// Element resource = entry.getNamedChild(RESOURCE); +// if (resource != null) { +// String et = resource.getType(); +// String eid = resource.getNamedChildValue(ID); +// if (t.equals(et) && i.equals(eid)) +// return entry; +// } +// } +// } } return null; } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java index c9d5a84d1..63a816ff2 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java @@ -37,6 +37,7 @@ import org.w3c.dom.Document; import java.io.*; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,6 +48,7 @@ public class IgLoader { private static final String[] IGNORED_EXTENSIONS = {"md", "css", "js", "png", "gif", "jpg", "html", "tgz", "pack", "zip"}; private static final String[] EXEMPT_FILES = {"spec.internals", "version.info", "schematron.zip", "package.json"}; + private static final int SCAN_HEADER_SIZE = 2048; @Getter private final FilesystemPackageCacheManager packageCacheManager; @Getter private final SimpleWorkerContext context; @@ -254,45 +256,87 @@ public class IgLoader { boolean recursive, VersionSourceInformation versions) throws Exception { Map source = loadIgSourceForVersion(src, recursive, true, versions); - if (source != null && source.containsKey("version.info")) - versions.see(readInfoVersion(source.get("version.info")), "version.info in " + src); + if (source != null) { + if (source.containsKey("version.info")) { + versions.see(readInfoVersion(source.get("version.info")), "version.info in " + src); + } else if (source.size() == 1) { + for (byte[] v : source.values()) { + scanForFhirVersion(versions, src, v); + } + } + } } public void scanForVersions(List sources, VersionSourceInformation versions) throws FHIRException, IOException { List refs = new ArrayList(); ValidatorUtils.parseSources(sources, refs, context); for (String ref : refs) { - Content cnt = loadContent(ref, "validate", false); - String s = TextFile.bytesToString(cnt.focus); - if (s.contains("http://hl7.org/fhir/3.0")) { - versions.see("3.0", "Profile in " + ref); - } - if (s.contains("http://hl7.org/fhir/1.0")) { - versions.see("1.0", "Profile in " + ref); - } - if (s.contains("http://hl7.org/fhir/4.0")) { - versions.see("4.0", "Profile in " + ref); - } - if (s.contains("http://hl7.org/fhir/1.4")) { - versions.see("1.4", "Profile in " + ref); - } - try { - if (s.startsWith("{")) { - JsonObject json = JsonTrackingParser.parse(s, null); - if (json.has("fhirVersion")) { - versions.see(VersionUtilities.getMajMin(JSONUtil.str(json, "fhirVersion")), "fhirVersion in " + ref); - } - } else { - Document doc = ValidatorUtils.parseXml(cnt.focus); - String v = XMLUtil.getNamedChildValue(doc.getDocumentElement(), "fhirVersion"); - if (v != null) { - versions.see(VersionUtilities.getMajMin(v), "fhirVersion in " + ref); + Content cnt = loadContent(ref, "validate", false); + scanForFhirVersion(versions, ref, cnt.focus); + } + } + + private void scanForFhirVersion(VersionSourceInformation versions, String ref, byte[] cnt) throws IOException { + String s = TextFile.bytesToString(cnt.length > SCAN_HEADER_SIZE ? Arrays.copyOfRange(cnt, 0, SCAN_HEADER_SIZE) : cnt).trim(); + try { + int i = s.indexOf("fhirVersion"); + if (i > 1) { + boolean xml = s.charAt(i) == '<'; + i = find(s, i, '"'); + if (!xml) { + i = find(s, i+1, '"'); + } + if (i > 0) { + int j = find(s, i+1, '"'); + if (j > 0) { + String v = s.substring(i+1, j); + if (VersionUtilities.isSemVer(v)) { + versions.see(VersionUtilities.getMajMin(v), "fhirVersion in " + ref); + return; + } + } + } + i = find(s, i, '\''); + if (!xml) { + i = find(s, i+1, '\''); + } + if (i > 0) { + int j = find(s, i+1, '\''); + if (j > 0) { + String v = s.substring(i, j); + if (VersionUtilities.isSemVer(v)) { + versions.see(VersionUtilities.getMajMin(v), "fhirVersion in " + ref); + return; + } } } - } catch (Exception e) { - // nothing } + } catch (Exception e) { + // nothing } + if (s.contains("http://hl7.org/fhir/3.0")) { + versions.see("3.0", "Profile in " + ref); + return; + } + if (s.contains("http://hl7.org/fhir/1.0")) { + versions.see("1.0", "Profile in " + ref); + return; + } + if (s.contains("http://hl7.org/fhir/4.0")) { + versions.see("4.0", "Profile in " + ref); + return; + } + if (s.contains("http://hl7.org/fhir/1.4")) { + versions.see("1.4", "Profile in " + ref); + return; + } + } + + private int find(String s, int i, char c) { + while (i < s.length() && s.charAt(i) != c) { + i++; + } + return i == s.length() ? -1 : i; } protected Map readZip(InputStream stream) throws IOException { diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ResourceChecker.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ResourceChecker.java index 9df74c7bc..1422b9810 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ResourceChecker.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ResourceChecker.java @@ -33,9 +33,9 @@ public class ResourceChecker { // return checkIsResource(context, debug, TextFile.fileToBytes(path), path); // } public static Manager.FhirFormat checkIsResource(SimpleWorkerContext context, boolean debug, byte[] cnt, String filename, boolean guessFromExtension) { - System.out.println(" ..Detect format for " + filename); +// System.out.println(" ..Detect format for " + filename); if (cnt.length == 0) { - System.out.println(" " + filename+" is empty"); + System.out.println("Loader: " + filename+" is empty"); return null; } if (guessFromExtension) { @@ -53,6 +53,9 @@ public class ResourceChecker { return Manager.FhirFormat.SHC; } if (Utilities.existsInList(ext, "json")) { + if (cnt.length > 2048) { + return FhirFormat.JSON; + } // no, we have to look inside, and decide. try { JsonObject json = JsonTrackingParser.parseJson(cnt); diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java index 49db7c3a5..16b6876e8 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java @@ -14,6 +14,7 @@ import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.r5.conformance.ProfileUtilities; import org.hl7.fhir.r5.context.IWorkerContext; import org.hl7.fhir.r5.context.IWorkerContext.ICanonicalResourceLocator; +import org.hl7.fhir.r5.context.IWorkerContext.IPackageLoadingTracker; import org.hl7.fhir.r5.context.IWorkerContext.PackageVersion; import org.hl7.fhir.r5.context.SimpleWorkerContext; import org.hl7.fhir.r5.context.SystemOutLoggingService; @@ -133,7 +134,7 @@ POSSIBILITY OF SUCH DAMAGE. * @author Grahame Grieve */ @Accessors(chain = true) -public class ValidationEngine implements IValidatorResourceFetcher, IValidationPolicyAdvisor, IPackageInstaller { +public class ValidationEngine implements IValidatorResourceFetcher, IValidationPolicyAdvisor, IPackageInstaller, IPackageLoadingTracker { @Getter @Setter private SimpleWorkerContext context; @Getter @Setter private Map binaries = new HashMap<>(); @@ -185,6 +186,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP * the validation framework in their own implementation context */ @Getter @Setter private Map validationControl = new HashMap<>(); + private Map resolvedUrls = new HashMap<>(); private ValidationEngine() { @@ -259,7 +261,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP ValidationEngine engine = new ValidationEngine(); engine.loadCoreDefinitions(src, false, terminologyCachePath, userAgent, timeTracker, loggingService); engine.getContext().setCanRunWithoutTerminology(canRunWithoutTerminologyServer); - + engine.getContext().setPackageTracker(engine); if (txServer != null) { engine.setTerminologyServer(txServer, txLog, txVersion); } @@ -429,7 +431,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP for (String ref : refs) { TimeTracker.Session tts = context.clock().start("validation"); context.clock().milestone(); - System.out.print(" Validate " + ref); + System.out.println(" Validate " + ref); Content cnt = igLoader.loadContent(ref, "validate", false); try { OperationOutcome outcome = validate(ref, cnt.focus, cnt.cntType, profiles, record); @@ -863,46 +865,49 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP @Override public boolean resolveURL(IResourceValidator validator, Object appContext, String path, String url, String type) throws FHIRException { + // some of this logic might take a while, and it's not going to change once loaded + if (resolvedUrls .containsKey(type+"|"+url)) { + return resolvedUrls.get(type+"|"+url); + } if (!url.startsWith("http://") && !url.startsWith("https://")) { // ignore these + resolvedUrls.put(type+"|"+url, true); return true; } - if (context.fetchResource(Resource.class, url) != null) + if (context.fetchResource(Resource.class, url) != null) { + resolvedUrls.put(type+"|"+url, true); return true; + } if (SIDUtilities.isKnownSID(url) || Utilities.existsInList(url, "http://hl7.org/fhir/w5", "http://hl7.org/fhir/fivews", "http://hl7.org/fhir/workflow", "http://hl7.org/fhir/ConsentPolicy/opt-out", "http://hl7.org/fhir/ConsentPolicy/opt-in")) { + resolvedUrls.put(type+"|"+url, true); return true; } if (Utilities.existsInList(url, "http://loinc.org", "http://unitsofmeasure.org", "http://snomed.info/sct")) { + resolvedUrls.put(type+"|"+url, true); return true; } - for (CanonicalResource cr : context.allConformanceResources()) { - if (cr instanceof NamingSystem) { - if (hasURL((NamingSystem) cr, url)) { - return true; - } - } + if (context.getNSUrlMap().containsKey(url)) { + resolvedUrls.put(type+"|"+url, true); + return true; } if (url.contains("example.org") || url.contains("acme.com")) { + resolvedUrls.put(type+"|"+url, false); return false; // todo... how to access settings from here? } if (fetcher != null) { try { - return fetcher.resolveURL(validator, appContext, path, url, type); + boolean ok = fetcher.resolveURL(validator, appContext, path, url, type); + resolvedUrls.put(type+"|"+url, ok); + return ok; } catch (Exception e) { + resolvedUrls.put(type+"|"+url, false); return false; } } + resolvedUrls.put(type+"|"+url, false); return false; } - private boolean hasURL(NamingSystem ns, String url) { - for (NamingSystemUniqueIdComponent uid : ns.getUniqueId()) { - if (uid.getType() == NamingSystemIdentifierType.URI && uid.hasValue() && uid.getValue().equals(url)) { - return true; - } - } - return false; - } @Override public CanonicalResource fetchCanonicalResource(IResourceValidator validator, String url) throws URISyntaxException { @@ -922,4 +927,10 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP return fetcher != null && fetcher.fetchesCanonicalResource(validator, url); } + @Override + public void packageLoaded(String pid, String version) { + resolvedUrls.clear(); + + } + } \ No newline at end of file diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/StandAloneValidatorFetcher.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/StandAloneValidatorFetcher.java index acb5caf2c..5e6510933 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/StandAloneValidatorFetcher.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/StandAloneValidatorFetcher.java @@ -248,7 +248,11 @@ public class StandAloneValidatorFetcher implements IValidatorResourceFetcher, IV @Override public CanonicalResource fetchCanonicalResource(IResourceValidator validator, String url) throws URISyntaxException { + if (url.contains("|")) { + url = url.substring(0, url.indexOf("|")); + } String[] p = url.split("\\/"); + String root = getRoot(p, url); if (root != null) { TerminologyClient c; diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidationService.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidationService.java index 35722bb1f..80c020537 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidationService.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/services/ValidationService.java @@ -19,6 +19,7 @@ import org.hl7.fhir.r5.renderers.spreadsheets.ValueSetSpreadsheetGenerator; import org.hl7.fhir.r5.terminologies.CodeSystemUtilities; import org.hl7.fhir.r5.utils.ToolingExtensions; import org.hl7.fhir.utilities.FhirPublication; +import org.hl7.fhir.utilities.SimpleTimeTracker; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.TimeTracker; import org.hl7.fhir.utilities.Utilities; @@ -361,7 +362,7 @@ public class ValidationService { validator.setCrumbTrails(cliContext.isCrumbTrails()); validator.setShowTimes(cliContext.isShowTimes()); validator.setAllowExampleUrls(cliContext.isAllowExampleUrls()); - StandAloneValidatorFetcher fetcher = new StandAloneValidatorFetcher(validator.getPcm(), validator.getContext(), validator); + StandAloneValidatorFetcher fetcher = new StandAloneValidatorFetcher(validator.getPcm(), validator.getContext(), validator); validator.setFetcher(fetcher); validator.getContext().setLocator(fetcher); validator.getBundleValidationRules().addAll(cliContext.getBundleValidationRules()); diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java index 1e8bcd951..ff10ddba6 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java @@ -143,6 +143,8 @@ import org.hl7.fhir.r5.utils.XVerExtensionManager; import org.hl7.fhir.r5.utils.validation.constants.*; import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; import org.hl7.fhir.utilities.SIDUtilities; +import org.hl7.fhir.utilities.SimpleTimeTracker; +import org.hl7.fhir.utilities.TimeTracker; import org.hl7.fhir.utilities.UnicodeUtilities; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.Utilities.DecimalStatus; @@ -159,6 +161,7 @@ import org.hl7.fhir.utilities.xhtml.XhtmlNode; import org.hl7.fhir.validation.BaseValidator; import org.hl7.fhir.validation.cli.utils.QuestionnaireMode; import org.hl7.fhir.validation.cli.utils.ValidationLevel; +import org.hl7.fhir.validation.instance.InstanceValidator.CanonicalResourceLookupResult; import org.hl7.fhir.validation.instance.type.BundleValidator; import org.hl7.fhir.validation.instance.type.CodeSystemValidator; import org.hl7.fhir.validation.instance.type.MeasureValidator; @@ -195,6 +198,20 @@ import com.google.gson.JsonObject; */ public class InstanceValidator extends BaseValidator implements IResourceValidator { + public class CanonicalResourceLookupResult { + + private CanonicalResource resource; + private String error; + + public CanonicalResourceLookupResult(CanonicalResource resource) { + this.resource = resource; + } + + public CanonicalResourceLookupResult(String error) { + this.error = error; + } + + } private static final String EXECUTED_CONSTRAINT_LIST = "validator.executed.invariant.list"; private static final String EXECUTION_ID = "validator.execution.id"; private static final String HTML_FRAGMENT_REGEX = "[a-zA-Z]\\w*(((\\s+)(\\S)*)*)"; @@ -301,18 +318,18 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat try { Element e = new ObjectConverter(context).convert((Resource) item); setParents(e); - self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage)); + self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage), null); } catch (IOException e1) { throw new FHIRException(e1); } } else if (item instanceof Element) { Element e = (Element) item; if (e.getSpecial() == SpecialElement.CONTAINED) { - self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e, ctxt.getRootResource(), ctxt.getGroupingResource()), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage)); + self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e, ctxt.getRootResource(), ctxt.getGroupingResource()), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage), null); } else if (e.getSpecial() != null) { - self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e, e, ctxt.getRootResource()), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage)); + self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e, e, ctxt.getRootResource()), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage), null); } else { - self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage)); + self.validateResource(new ValidatorHostContext(ctxt.getAppContext(), e), valerrors, e, e, sd, IdStatus.OPTIONAL, new NodeStack(context, null, e, validationLanguage), null); } } else throw new NotImplementedException(context.formatMessage(I18nConstants.NOT_DONE_YET_VALIDATORHOSTSERVICESCONFORMSTOPROFILE_WHEN_ITEM_IS_NOT_AN_ELEMENT)); @@ -387,6 +404,8 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat private IValidatorResourceFetcher fetcher; private IValidationPolicyAdvisor policyAdvisor; long time = 0; + long start = 0; + long lastlog = 0; private IEvaluationContext externalHostServices; private boolean noExtensibleWarnings; private String serverBase; @@ -404,9 +423,11 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat private boolean validateValueSetCodesOnTxServer = true; private QuestionnaireMode questionnaireMode; private ValidationOptions baseOptions = new ValidationOptions(); + private Map crLookups = new HashMap<>(); public InstanceValidator(IWorkerContext theContext, IEvaluationContext hostServices, XVerExtensionManager xverManager) { super(theContext, xverManager); + start = System.currentTimeMillis(); this.externalHostServices = hostServices; this.profileUtilities = new ProfileUtilities(theContext, null, null); fpe = new FHIRPathEngine(context); @@ -761,10 +782,10 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat long t = System.nanoTime(); if (profiles == null || profiles.isEmpty()) { - validateResource(new ValidatorHostContext(appContext, element), errors, element, element, null, resourceIdRule, new NodeStack(context, path, element, validationLanguage).resetIds()); + validateResource(new ValidatorHostContext(appContext, element), errors, element, element, null, resourceIdRule, new NodeStack(context, path, element, validationLanguage).resetIds(), null); } else { for (StructureDefinition defn : profiles) { - validateResource(new ValidatorHostContext(appContext, element), errors, element, element, defn, resourceIdRule, new NodeStack(context, path, element, validationLanguage).resetIds()); + validateResource(new ValidatorHostContext(appContext, element), errors, element, element, defn, resourceIdRule, new NodeStack(context, path, element, validationLanguage).resetIds(), null); } } if (hintAboutNonMustSupport) { @@ -1669,7 +1690,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } - private StructureDefinition checkExtension(ValidatorHostContext hostContext, List errors, String path, Element resource, Element container, Element element, ElementDefinition def, StructureDefinition profile, NodeStack stack, NodeStack containerStack, String extensionUrl) throws FHIRException { + private StructureDefinition checkExtension(ValidatorHostContext hostContext, List errors, String path, Element resource, Element container, Element element, ElementDefinition def, StructureDefinition profile, NodeStack stack, NodeStack containerStack, String extensionUrl, PercentageTracker pct) throws FHIRException { String url = element.getNamedChildValue("url"); boolean isModifier = element.getName().equals("modifierExtension"); assert def.getIsModifier() == isModifier; @@ -1719,7 +1740,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat rule(errors, IssueType.STRUCTURE, element.line(), element.col(), path, allowedTypes.contains(actualType), I18nConstants.EXTENSION_EXT_TYPE, url, allowedTypes.toString(), actualType); // 3. is the content of the extension valid? - validateElement(hostContext, errors, ex, ex.getSnapshot().getElement().get(0), null, null, resource, element, "Extension", stack, false, true, url); + validateElement(hostContext, errors, ex, ex.getSnapshot().getElement().get(0), null, null, resource, element, "Extension", stack, false, true, url, pct); } return ex; @@ -2926,7 +2947,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat StructureDefinition profile, ElementDefinition container, String parentType, - NodeStack stack) throws FHIRException { + NodeStack stack, PercentageTracker pct) throws FHIRException { Reference reference = ObjectConverter.readAsReference(element); String ref = reference.getReference(); @@ -3058,7 +3079,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat for (StructureDefinition pr : profiles) { List profileErrors = new ArrayList(); validateResource(we.hostContext(hostContext, pr), profileErrors, we.getResource(), we.getFocus(), pr, - IdStatus.OPTIONAL, we.getStack().resetIds()); + IdStatus.OPTIONAL, we.getStack().resetIds(), pct); if (!hasErrors(profileErrors)) { goodCount++; goodProfiles.put(pr, profileErrors); @@ -4366,17 +4387,29 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } // checkSpecials = we're only going to run these tests if we are actually validating this content (as opposed to we looked it up) - private void start(ValidatorHostContext hostContext, List errors, Element resource, Element element, StructureDefinition defn, NodeStack stack) throws FHIRException { + private void start(ValidatorHostContext hostContext, List errors, Element resource, Element element, StructureDefinition defn, NodeStack stack, PercentageTracker pct) throws FHIRException { checkLang(resource, stack); if (crumbTrails) { element.addMessage(signpost(errors, IssueType.INFORMATIONAL, element.line(), element.col(), stack.getLiteralPath(), I18nConstants.VALIDATION_VAL_PROFILE_SIGNPOST, defn.getUrl())); } - + boolean pctOwned = false; + if (pct == null) { + // this method is reentrant, but also the right place to tell the user what is going on if it's the root. + // if we're not at the root, we don't report progress + pctOwned = true; + pct = new PercentageTracker(resource.countDescendents(), resource.fhirType(), defn.getUrl()); + } if (BUNDLE.equals(element.fhirType())) { + if (debug) { + System.out.println("Resolve Bundle Entries "+time()); + } resolveBundleReferences(element, new ArrayList()); } - startInner(hostContext, errors, resource, element, defn, stack, hostContext.isCheckSpecials()); - + startInner(hostContext, errors, resource, element, defn, stack, hostContext.isCheckSpecials(), pct); + if (pctOwned) { + pct.done(); + } + Element meta = element.getNamedChild(META); if (meta != null) { List profiles = new ArrayList(); @@ -4404,14 +4437,27 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } else if (!fetcher.fetchesCanonicalResource(this, profile.primitiveValue())) { warning(errors, IssueType.STRUCTURE, element.line(), element.col(), stack.getLiteralPath() + ".meta.profile[" + i + "]", false, I18nConstants.VALIDATION_VAL_PROFILE_UNKNOWN_NOT_POLICY, profile.primitiveValue()); } else { - try { - sd = (StructureDefinition) fetcher.fetchCanonicalResource(this, profile.primitiveValue()); - } catch (Exception e) { - if (STACK_TRACE) e.printStackTrace(); - warning(errors, IssueType.STRUCTURE, element.line(), element.col(), stack.getLiteralPath() + ".meta.profile[" + i + "]", false, I18nConstants.VALIDATION_VAL_PROFILE_UNKNOWN_ERROR, profile.primitiveValue(), e.getMessage()); - } - if (sd != null) { - context.cacheResource(sd); + sd = null; + String url = profile.primitiveValue(); + CanonicalResourceLookupResult cr = crLookups.get(url); + if (cr != null) { + if (cr.error != null) { + warning(errors, IssueType.STRUCTURE, element.line(), element.col(), stack.getLiteralPath() + ".meta.profile[" + i + "]", false, I18nConstants.VALIDATION_VAL_PROFILE_UNKNOWN_ERROR, url, cr.error); + } else { + sd = (StructureDefinition) cr.resource; + } + } else { + try { + sd = (StructureDefinition) fetcher.fetchCanonicalResource(this, url); + crLookups.put(url, new CanonicalResourceLookupResult(sd)); + } catch (Exception e) { + if (STACK_TRACE) { e.printStackTrace(); } + crLookups.put(url, new CanonicalResourceLookupResult(e.getMessage())); + warning(errors, IssueType.STRUCTURE, element.line(), element.col(), stack.getLiteralPath() + ".meta.profile[" + i + "]", false, I18nConstants.VALIDATION_VAL_PROFILE_UNKNOWN_ERROR, profile.primitiveValue(), e.getMessage()); + } + if (sd != null) { + context.cacheResource(sd); + } } } } @@ -4420,7 +4466,13 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat element.addMessage(signpost(errors, IssueType.INFORMATIONAL, element.line(), element.col(), stack.getLiteralPath(), I18nConstants.VALIDATION_VAL_PROFILE_SIGNPOST_META, sd.getUrl())); } stack.resetIds(); - startInner(hostContext, errors, resource, element, sd, stack, false); + if (pctOwned) { + pct = new PercentageTracker(resource.countDescendents(), resource.fhirType(), sd.getUrl()); + } + startInner(hostContext, errors, resource, element, sd, stack, false, pct); + if (pctOwned) { + pct.done(); + } } } } @@ -4437,13 +4489,28 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat element.addMessage(signpost(errors, IssueType.INFORMATIONAL, element.line(), element.col(), stack.getLiteralPath(), I18nConstants.VALIDATION_VAL_PROFILE_SIGNPOST_GLOBAL, sd.getUrl(), ig.getUrl())); } stack.resetIds(); - startInner(hostContext, errors, resource, element, sd, stack, false); + if (pctOwned) { + pct = new PercentageTracker(resource.countDescendents(), resource.fhirType(), sd.getUrl()); + } + startInner(hostContext, errors, resource, element, sd, stack, false, pct); + if (pctOwned) { + pct.done(); + } } } } } +// System.out.println("start: "+(System.currentTimeMillis()-st)+" ("+resource.fhirType()+")"); } +// private void plog(String msg) { +// long n = System.currentTimeMillis(); +// String elapsed = Utilities.padLeft(Long.toString(n-start), ' ', 5); +// String delta = Utilities.padLeft(lastlog == 0 ? "0" : Long.toString(n-lastlog), ' ', 5); +// lastlog = n; +// System.out.println("-- "+elapsed+" "+delta+" "+msg); +// } + private void resolveBundleReferences(Element element, List bundles) { if (!element.hasUserData("validator.bundle.resolved")) { element.setUserData("validator.bundle.resolved", true); @@ -4462,7 +4529,6 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } private void resolveBundleReferencesInResource(List bundles, Element r, String fu) { - r.setUserData("validator.bundle.resolution-resource", null); if (BUNDLE.equals(r.fhirType())) { resolveBundleReferences(r, bundles); } else { @@ -4478,16 +4544,14 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat if (!Utilities.noString(ref)) { for (Element bundle : bundles) { List entries = bundle.getChildren(ENTRY); - Element tgt = resolveInBundle(entries, ref, fu, resource.fhirType(), resource.getIdBase()); + Element tgt = resolveInBundle(bundle, entries, ref, fu, resource.fhirType(), resource.getIdBase()); if (tgt != null) { element.setUserData("validator.bundle.resolution", tgt.getNamedChild(RESOURCE)); return; } } - element.setUserData("validator.bundle.resolution-failed", ref); } } else { - element.setUserData("validator.bundle.resolution-noref", null); for (Element child : element.getChildren()) { resolveBundleReferencesForElement(bundles, resource, fu, child); } @@ -4495,7 +4559,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } - public void startInner(ValidatorHostContext hostContext, List errors, Element resource, Element element, StructureDefinition defn, NodeStack stack, boolean checkSpecials) { + public void startInner(ValidatorHostContext hostContext, List errors, Element resource, Element element, StructureDefinition defn, NodeStack stack, boolean checkSpecials, PercentageTracker pct) { // the first piece of business is to see if we've validated this resource against this profile before. // if we have (*or if we still are*), then we'll just return our existing errors ResourceValidationTracker resTracker = getResourceTracker(element); @@ -4512,7 +4576,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat List localErrors = new ArrayList(); resTracker.startValidating(defn); trackUsage(defn, hostContext, element); - validateElement(hostContext, localErrors, defn, defn.getSnapshot().getElement().get(0), null, null, resource, element, element.getName(), stack, false, true, null); + validateElement(hostContext, localErrors, defn, defn.getSnapshot().getElement().get(0), null, null, resource, element, element.getName(), stack, false, true, null, pct); resTracker.storeOutcomes(defn, localErrors); for (ValidationMessage vm : localErrors) { if (!errors.contains(vm)) { @@ -4521,15 +4585,15 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } } if (checkSpecials) { - checkSpecials(hostContext, errors, element, stack, checkSpecials); + checkSpecials(hostContext, errors, element, stack, checkSpecials, pct); validateResourceRules(errors, element, stack); } } - public void checkSpecials(ValidatorHostContext hostContext, List errors, Element element, NodeStack stack, boolean checkSpecials) { + public void checkSpecials(ValidatorHostContext hostContext, List errors, Element element, NodeStack stack, boolean checkSpecials, PercentageTracker pct) { // specific known special validations if (element.getType().equals(BUNDLE)) { - new BundleValidator(context, serverBase, this, xverManager, jurisdiction).validateBundle(errors, element, stack, checkSpecials, hostContext); + new BundleValidator(context, serverBase, this, xverManager, jurisdiction).validateBundle(errors, element, stack, checkSpecials, hostContext, pct); } else if (element.getType().equals("Observation")) { validateObservation(errors, element, stack); } else if (element.getType().equals("Questionnaire")) { @@ -4635,7 +4699,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat private void validateContains(ValidatorHostContext hostContext, List errors, String path, ElementDefinition child, ElementDefinition context, Element resource, - Element element, NodeStack stack, IdStatus idstatus, StructureDefinition parentProfile) throws FHIRException { + Element element, NodeStack stack, IdStatus idstatus, StructureDefinition parentProfile, PercentageTracker pct) throws FHIRException { SpecialElement special = element.getSpecial(); @@ -4701,7 +4765,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat trackUsage(profile, hostContext, element); if (rule(errors, IssueType.INVALID, element.line(), element.col(), stack.getLiteralPath(), profile != null, I18nConstants.BUNDLE_BUNDLE_ENTRY_NOPROFILE_EXPL, special.toHuman(), resourceName, typeForResource.getProfile().get(0).asStringValue())) { - validateResource(hc, errors, resource, element, profile, idstatus, stack); + validateResource(hc, errors, resource, element, profile, idstatus, stack, pct); } } else if (typeForResource.getProfile().isEmpty()) { long t = System.nanoTime(); @@ -4711,7 +4775,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat trackUsage(profile, hostContext, element); if (rule(errors, IssueType.INVALID, element.line(), element.col(), stack.getLiteralPath(), profile != null, I18nConstants.BUNDLE_BUNDLE_ENTRY_NOPROFILE_TYPE, special == null ? "??" : special.toHuman(), resourceName)) { - validateResource(hc, errors, resource, element, profile, idstatus, stack); + validateResource(hc, errors, resource, element, profile, idstatus, stack, pct); } } else { CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); @@ -4774,8 +4838,10 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat private void validateElement(ValidatorHostContext hostContext, List errors, StructureDefinition profile, ElementDefinition definition, StructureDefinition cprofile, ElementDefinition context, - Element resource, Element element, String actualType, NodeStack stack, boolean inCodeableConcept, boolean checkDisplayInContext, String extensionUrl) throws FHIRException { + Element resource, Element element, String actualType, NodeStack stack, boolean inCodeableConcept, boolean checkDisplayInContext, String extensionUrl, PercentageTracker pct) throws FHIRException { + pct.seeElement(element); + String id = element.getChildValue("id"); if (!Utilities.noString(id)) { if (stack.getIds().containsKey(id) && stack.getIds().get(id) != element) { @@ -4797,8 +4863,8 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } if (definition.getPattern() != null) { checkFixedValue(errors, stack.getLiteralPath(), element, definition.getPattern(), profile.getUrl(), definition.getSliceName(), null, true); - } - + } + // get the list of direct defined children, including slices List childDefinitions = profileUtilities.getChildMap(profile, definition); if (childDefinitions.isEmpty()) { @@ -4819,10 +4885,10 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat checkCardinalities(errors, profile, element, stack, childDefinitions, children, problematicPaths); // 4. check order if any slices are ordered. (todo) - + // 5. inspect each child for validity for (ElementInfo ei : children) { - checkChild(hostContext, errors, profile, definition, resource, element, actualType, stack, inCodeableConcept, checkDisplayInContext, ei, extensionUrl); + checkChild(hostContext, errors, profile, definition, resource, element, actualType, stack, inCodeableConcept, checkDisplayInContext, ei, extensionUrl, pct); } } @@ -4860,7 +4926,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } public void checkChild(ValidatorHostContext hostContext, List errors, StructureDefinition profile, ElementDefinition definition, - Element resource, Element element, String actualType, NodeStack stack, boolean inCodeableConcept, boolean checkDisplayInContext, ElementInfo ei, String extensionUrl) + Element resource, Element element, String actualType, NodeStack stack, boolean inCodeableConcept, boolean checkDisplayInContext, ElementInfo ei, String extensionUrl, PercentageTracker pct) throws FHIRException, DefinitionException { if (debug && ei.definition != null && ei.slice != null) { @@ -4868,25 +4934,33 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } if (ei.definition != null) { if (debug) { - System.out.println(Utilities.padLeft("", ' ', stack.depth())+ "Check "+ei.getPath()+" against defn "+ei.definition.getId()+" from "+profile.getUrl()); + System.out.println(Utilities.padLeft("", ' ', stack.depth())+ "Check "+ei.getPath()+" against defn "+ei.definition.getId()+" from "+profile.getUrl()+time()); } - checkChildByDefinition(hostContext, errors, profile, definition, resource, element, actualType, stack, inCodeableConcept, checkDisplayInContext, ei, extensionUrl, ei.definition, false); + checkChildByDefinition(hostContext, errors, profile, definition, resource, element, actualType, stack, inCodeableConcept, checkDisplayInContext, ei, extensionUrl, ei.definition, false, pct); } if (ei.slice != null) { if (debug) { - System.out.println(Utilities.padLeft("", ' ', stack.depth())+ "Check "+ei.getPath()+" against slice "+ei.slice.getId()); + System.out.println(Utilities.padLeft("", ' ', stack.depth())+ "Check "+ei.getPath()+" against slice "+ei.slice.getId()+time()); } - checkChildByDefinition(hostContext, errors, profile, definition, resource, element, actualType, stack, inCodeableConcept, checkDisplayInContext, ei, extensionUrl, ei.slice, true); + checkChildByDefinition(hostContext, errors, profile, definition, resource, element, actualType, stack, inCodeableConcept, checkDisplayInContext, ei, extensionUrl, ei.slice, true, pct); } } + private String time() { + long t = System.currentTimeMillis(); + String s = " "+(t - start)+"ms"; + start = t; + return s; + } + public void checkChildByDefinition(ValidatorHostContext hostContext, List errors, StructureDefinition profile, ElementDefinition definition, Element resource, Element element, String actualType, NodeStack stack, boolean inCodeableConcept, - boolean checkDisplayInContext, ElementInfo ei, String extensionUrl, ElementDefinition checkDefn, boolean isSlice) { + boolean checkDisplayInContext, ElementInfo ei, String extensionUrl, ElementDefinition checkDefn, boolean isSlice, PercentageTracker pct) { List profiles = new ArrayList(); String type = null; ElementDefinition typeDefn = null; checkMustSupport(profile, ei); + long s = System.currentTimeMillis(); if (checkDefn.getType().size() == 1 && !"*".equals(checkDefn.getType().get(0).getWorkingCode()) && !"Element".equals(checkDefn.getType().get(0).getWorkingCode()) && !"BackboneElement".equals(checkDefn.getType().get(0).getWorkingCode())) { @@ -4968,9 +5042,9 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } } NodeStack localStack = stack.push(ei.getElement(), "*".equals(ei.getDefinition().getBase().getMax()) && ei.count == -1 ? 0 : ei.count, checkDefn, type == null ? typeDefn : resolveType(type, checkDefn.getType())); -// if (debug) { -// System.out.println(" check " + localStack.getLiteralPath()+" against "+ei.getDefinition().getId()+" in profile "+profile.getUrl()); -// } + if (debug) { + System.out.println(" check " + localStack.getLiteralPath()+" against "+ei.getDefinition().getId()+" in profile "+profile.getUrl()+time()); + } String localStackLiteralPath = localStack.getLiteralPath(); String eiPath = ei.getPath(); if (!eiPath.equals(localStackLiteralPath)) { @@ -5012,7 +5086,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat checkDisplay = checkCodeableConcept(errors, ei.getPath(), ei.getElement(), profile, checkDefn, stack); thisIsCodeableConcept = true; } else if (type.equals("Reference")) { - checkReference(hostContext, errors, ei.getPath(), ei.getElement(), profile, checkDefn, actualType, localStack); + checkReference(hostContext, errors, ei.getPath(), ei.getElement(), profile, checkDefn, actualType, localStack, pct); // We only check extensions if we're not in a complex extension or if the element we're dealing with is not defined as part of that complex extension } else if (type.equals("Extension")) { Element eurl = ei.getElement().getNamedChild("url"); @@ -5021,13 +5095,13 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat thisExtension = url; if (rule(errors, IssueType.INVALID, ei.getPath(), !Utilities.noString(url), I18nConstants.EXTENSION_EXT_URL_NOTFOUND)) { if (rule(errors, IssueType.INVALID, ei.getPath(), (extensionUrl != null) || Utilities.isAbsoluteUrl(url), I18nConstants.EXTENSION_EXT_URL_ABSOLUTE)) { - checkExtension(hostContext, errors, ei.getPath(), resource, element, ei.getElement(), checkDefn, profile, localStack, stack, extensionUrl); + checkExtension(hostContext, errors, ei.getPath(), resource, element, ei.getElement(), checkDefn, profile, localStack, stack, extensionUrl, pct); } } } } else if (type.equals("Resource") || isResource(type)) { validateContains(hostContext, errors, ei.getPath(), checkDefn, definition, resource, ei.getElement(), - localStack, idStatusForEntry(element, ei), profile); // if + localStack, idStatusForEntry(element, ei), profile, pct); // if elementValidated = true; // (str.matches(".*([.,/])work\\1$")) } else if (Utilities.isAbsoluteUrl(type)) { @@ -5044,7 +5118,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } } else { if (rule(errors, IssueType.STRUCTURE, ei.line(), ei.col(), stack.getLiteralPath(), checkDefn != null, I18nConstants.VALIDATION_VAL_CONTENT_UNKNOWN, ei.getName())) - validateElement(hostContext, errors, profile, checkDefn, null, null, resource, ei.getElement(), type, localStack, false, true, null); + validateElement(hostContext, errors, profile, checkDefn, null, null, resource, ei.getElement(), type, localStack, false, true, null, pct); } StructureDefinition p = null; String tail = null; @@ -5083,7 +5157,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat p = this.context.fetchResource(StructureDefinition.class, typeProfile); if (rule(errors, IssueType.STRUCTURE, ei.line(), ei.col(), ei.getPath(), p != null, I18nConstants.VALIDATION_VAL_UNKNOWN_PROFILE, typeProfile)) { List profileErrors = new ArrayList(); - validateElement(hostContext, profileErrors, p, getElementByTail(p, tail), profile, checkDefn, resource, ei.getElement(), type, localStack, thisIsCodeableConcept, checkDisplay, thisExtension); + validateElement(hostContext, profileErrors, p, getElementByTail(p, tail), profile, checkDefn, resource, ei.getElement(), type, localStack, thisIsCodeableConcept, checkDisplay, thisExtension, pct); if (hasErrors(profileErrors)) badProfiles.put(typeProfile, profileErrors); else @@ -5117,15 +5191,15 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat if (!elementValidated) { if (ei.getElement().getSpecial() == SpecialElement.BUNDLE_ENTRY || ei.getElement().getSpecial() == SpecialElement.BUNDLE_OUTCOME || ei.getElement().getSpecial() == SpecialElement.PARAMETER) - validateElement(hostContext, errors, p, getElementByTail(p, tail), profile, checkDefn, ei.getElement(), ei.getElement(), type, localStack.resetIds(), thisIsCodeableConcept, checkDisplay, thisExtension); + validateElement(hostContext, errors, p, getElementByTail(p, tail), profile, checkDefn, ei.getElement(), ei.getElement(), type, localStack.resetIds(), thisIsCodeableConcept, checkDisplay, thisExtension, pct); else - validateElement(hostContext, errors, p, getElementByTail(p, tail), profile, checkDefn, resource, ei.getElement(), type, localStack, thisIsCodeableConcept, checkDisplay, thisExtension); + validateElement(hostContext, errors, p, getElementByTail(p, tail), profile, checkDefn, resource, ei.getElement(), type, localStack, thisIsCodeableConcept, checkDisplay, thisExtension, pct); } int index = profile.getSnapshot().getElement().indexOf(checkDefn); if (index < profile.getSnapshot().getElement().size() - 1) { String nextPath = profile.getSnapshot().getElement().get(index + 1).getPath(); if (!nextPath.equals(checkDefn.getPath()) && nextPath.startsWith(checkDefn.getPath())) - validateElement(hostContext, errors, profile, checkDefn, null, null, resource, ei.getElement(), type, localStack, thisIsCodeableConcept, checkDisplay, thisExtension); + validateElement(hostContext, errors, profile, checkDefn, null, null, resource, ei.getElement(), type, localStack, thisIsCodeableConcept, checkDisplay, thisExtension, pct); } } } @@ -5357,8 +5431,9 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat // 1. List the children, and remember their exact path (convenience) List children = new ArrayList(); ChildIterator iter = new ChildIterator(this, stack.getLiteralPath(), element); - while (iter.next()) + while (iter.next()) { children.add(new ElementInfo(iter.name(), iter.element(), iter.path(), iter.count())); + } return children; } @@ -5535,9 +5610,9 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat } public void checkInvariant(ValidatorHostContext hostContext, List errors, String path, StructureDefinition profile, Element resource, Element element, ElementDefinitionConstraintComponent inv) throws FHIRException { -// if (debug) { -// System.out.println("inv "+inv.getKey()+" on "+path+" in "+resource.fhirType()+" {{ "+inv.getExpression()+" }}"); -// } + if (debug) { + System.out.println("inv "+inv.getKey()+" on "+path+" in "+resource.fhirType()+" {{ "+inv.getExpression()+" }}"+time()); + } ExpressionNode n = (ExpressionNode) inv.getUserData("validator.expression.cache"); if (n == null) { long t = System.nanoTime(); @@ -5600,7 +5675,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat * The actual base entry point for internal use (re-entrant) */ private void validateResource(ValidatorHostContext hostContext, List errors, Element resource, - Element element, StructureDefinition defn, IdStatus idstatus, NodeStack stack) throws FHIRException { + Element element, StructureDefinition defn, IdStatus idstatus, NodeStack stack, PercentageTracker pct) throws FHIRException { // check here if we call validation policy here, and then change it to the new interface assert stack != null; @@ -5646,7 +5721,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat // validate if (rule(errors, IssueType.INVALID, element.line(), element.col(), stack.getLiteralPath(), resourceName.equals(defn.getType()), I18nConstants.VALIDATION_VAL_PROFILE_WRONGTYPE, defn.getType(), resourceName, defn.getUrl())) { - start(hostContext, errors, element, element, defn, stack); // root is both definition and type + start(hostContext, errors, element, element, defn, stack, pct); // root is both definition and type } } } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java new file mode 100644 index 000000000..536ce0d38 --- /dev/null +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java @@ -0,0 +1,38 @@ +package org.hl7.fhir.validation.instance; + +import org.hl7.fhir.r5.elementmodel.Element; + +public class PercentageTracker { + + private int total; + private int last; + private int current; + + private static int instance; + + public PercentageTracker(int total, String fhirType, String url) { + this.total = total; + instance++; + last = 0; + System.out.print("Validate "+fhirType+" against "+url); + } + + public void done() { + System.out.println("|"); + } + + public void seeElement(Element e) { + if (e.getInstanceId() != instance) { + e.setInstanceId(instance); + current++; + int pct = (current*100) / total; + if (pct > last + 5) { + while (last + 5 < pct) { + System.out.print("."); + last = last + 5; + } + } + } + } + +} diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/type/BundleValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/type/BundleValidator.java index f3e4effbb..eb5a162aa 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/type/BundleValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/type/BundleValidator.java @@ -23,6 +23,7 @@ import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType; import org.hl7.fhir.validation.BaseValidator; import org.hl7.fhir.validation.instance.InstanceValidator; +import org.hl7.fhir.validation.instance.PercentageTracker; import org.hl7.fhir.validation.instance.utils.EntrySummary; import org.hl7.fhir.validation.instance.utils.IndexedElement; import org.hl7.fhir.validation.instance.utils.NodeStack; @@ -40,7 +41,7 @@ public class BundleValidator extends BaseValidator { this.jurisdiction = jurisdiction; } - public void validateBundle(List errors, Element bundle, NodeStack stack, boolean checkSpecials, ValidatorHostContext hostContext) { + public void validateBundle(List errors, Element bundle, NodeStack stack, boolean checkSpecials, ValidatorHostContext hostContext, PercentageTracker pct) { List entries = new ArrayList(); bundle.getNamedChildren(ENTRY, entries); String type = bundle.getNamedChildValue(TYPE); @@ -60,7 +61,7 @@ public class BundleValidator extends BaseValidator { Element resource = firstEntry.getNamedChild(RESOURCE); if (rule(errors, IssueType.INVALID, firstEntry.line(), firstEntry.col(), stack.addToLiteralPath(ENTRY, PATH_ARG), resource != null, I18nConstants.BUNDLE_BUNDLE_ENTRY_NOFIRSTRESOURCE)) { String id = resource.getNamedChildValue(ID); - validateDocument(errors, entries, resource, firstStack.push(resource, -1, null, null), fullUrl, id); + validateDocument(errors, bundle, entries, resource, firstStack.push(resource, -1, null, null), fullUrl, id); } if (!VersionUtilities.isThisOrLater(FHIRVersion._4_0_1.getDisplay(), bundle.getProperty().getStructure().getFhirVersion().getDisplay())) { handleSpecialCaseForLastUpdated(bundle, errors, stack); @@ -118,7 +119,7 @@ public class BundleValidator extends BaseValidator { res.addMessage(signpost(errors, IssueType.INFORMATIONAL, res.line(), res.col(), stack.getLiteralPath(), I18nConstants.VALIDATION_VAL_PROFILE_SIGNPOST_BUNDLE_PARAM, defn.getUrl())); } stack.resetIds(); - validator.startInner(hostContext, errors, res, res, defn, rstack, false); + validator.startInner(hostContext, errors, res, res, defn, rstack, false, pct); } } } @@ -282,23 +283,23 @@ public class BundleValidator extends BaseValidator { return null; } - private void validateDocument(List errors, List entries, Element composition, NodeStack stack, String fullUrl, String id) { + private void validateDocument(List errors, Element bundle, List entries, Element composition, NodeStack stack, String fullUrl, String id) { // first entry must be a composition if (rule(errors, IssueType.INVALID, composition.line(), composition.col(), stack.getLiteralPath(), composition.getType().equals("Composition"), I18nConstants.BUNDLE_BUNDLE_ENTRY_DOCUMENT)) { // the composition subject etc references must resolve in the bundle - validateDocumentReference(errors, entries, composition, stack, fullUrl, id, false, "subject", "Composition"); - validateDocumentReference(errors, entries, composition, stack, fullUrl, id, true, "author", "Composition"); - validateDocumentReference(errors, entries, composition, stack, fullUrl, id, false, "encounter", "Composition"); - validateDocumentReference(errors, entries, composition, stack, fullUrl, id, false, "custodian", "Composition"); - validateDocumentSubReference(errors, entries, composition, stack, fullUrl, id, "Composition", "attester", false, "party"); - validateDocumentSubReference(errors, entries, composition, stack, fullUrl, id, "Composition", "event", true, "detail"); + validateDocumentReference(errors, bundle, entries, composition, stack, fullUrl, id, false, "subject", "Composition"); + validateDocumentReference(errors, bundle, entries, composition, stack, fullUrl, id, true, "author", "Composition"); + validateDocumentReference(errors, bundle, entries, composition, stack, fullUrl, id, false, "encounter", "Composition"); + validateDocumentReference(errors, bundle, entries, composition, stack, fullUrl, id, false, "custodian", "Composition"); + validateDocumentSubReference(errors, bundle, entries, composition, stack, fullUrl, id, "Composition", "attester", false, "party"); + validateDocumentSubReference(errors, bundle, entries, composition, stack, fullUrl, id, "Composition", "event", true, "detail"); - validateSections(errors, entries, composition, stack, fullUrl, id); + validateSections(errors, bundle, entries, composition, stack, fullUrl, id); } } - private void validateSections(List errors, List entries, Element focus, NodeStack stack, String fullUrl, String id) { + private void validateSections(List errors, Element bundle, List entries, Element focus, NodeStack stack, String fullUrl, String id) { List sections = new ArrayList(); focus.getNamedChildren("section", sections); int i = 1; @@ -306,48 +307,48 @@ public class BundleValidator extends BaseValidator { NodeStack localStack = stack.push(section, i, null, null); // technically R4+, but there won't be matches from before that - validateDocumentReference(errors, entries, section, stack, fullUrl, id, true, "author", "Section"); - validateDocumentReference(errors, entries, section, stack, fullUrl, id, false, "focus", "Section"); + validateDocumentReference(errors, bundle, entries, section, stack, fullUrl, id, true, "author", "Section"); + validateDocumentReference(errors, bundle, entries, section, stack, fullUrl, id, false, "focus", "Section"); List sectionEntries = new ArrayList(); section.getNamedChildren(ENTRY, sectionEntries); int j = 1; for (Element sectionEntry : sectionEntries) { NodeStack localStack2 = localStack.push(sectionEntry, j, null, null); - validateBundleReference(errors, entries, sectionEntry, "Section Entry", localStack2, fullUrl, "Composition", id); + validateBundleReference(errors, bundle, entries, sectionEntry, "Section Entry", localStack2, fullUrl, "Composition", id); j++; } - validateSections(errors, entries, section, localStack, fullUrl, id); + validateSections(errors, bundle, entries, section, localStack, fullUrl, id); i++; } } - public void validateDocumentSubReference(List errors, List entries, Element composition, NodeStack stack, String fullUrl, String id, String title, String parent, boolean repeats, String propName) { + public void validateDocumentSubReference(List errors, Element bundle, List entries, Element composition, NodeStack stack, String fullUrl, String id, String title, String parent, boolean repeats, String propName) { List list = new ArrayList<>(); composition.getNamedChildren(parent, list); int i = 1; for (Element elem : list) { - validateDocumentReference(errors, entries, elem, stack.push(elem, i, null, null), fullUrl, id, repeats, propName, title + "." + parent); + validateDocumentReference(errors, bundle, entries, elem, stack.push(elem, i, null, null), fullUrl, id, repeats, propName, title + "." + parent); i++; } } - public void validateDocumentReference(List errors, List entries, Element composition, NodeStack stack, String fullUrl, String id, boolean repeats, String propName, String title) { + public void validateDocumentReference(List errors, Element bundle, List entries, Element composition, NodeStack stack, String fullUrl, String id, boolean repeats, String propName, String title) { if (repeats) { List list = new ArrayList<>(); composition.getNamedChildren(propName, list); int i = 1; for (Element elem : list) { - validateBundleReference(errors, entries, elem, title + "." + propName, stack.push(elem, i, null, null), fullUrl, "Composition", id); + validateBundleReference(errors, bundle, entries, elem, title + "." + propName, stack.push(elem, i, null, null), fullUrl, "Composition", id); i++; } } else { Element elem = composition.getNamedChild(propName); if (elem != null) { - validateBundleReference(errors, entries, elem, title + "." + propName, stack.push(elem, -1, null, null), fullUrl, "Composition", id); + validateBundleReference(errors, bundle, entries, elem, title + "." + propName, stack.push(elem, -1, null, null), fullUrl, "Composition", id); } } } @@ -357,11 +358,11 @@ public class BundleValidator extends BaseValidator { if (rule(errors, IssueType.INVALID, messageHeader.line(), messageHeader.col(), stack.getLiteralPath(), messageHeader.getType().equals("MessageHeader"), I18nConstants.VALIDATION_BUNDLE_MESSAGE)) { List elements = messageHeader.getChildren("focus"); for (Element elem : elements) - validateBundleReference(errors, entries, elem, "MessageHeader Data", stack.push(elem, -1, null, null), fullUrl, "MessageHeader", id); + validateBundleReference(errors, messageHeader, entries, elem, "MessageHeader Data", stack.push(elem, -1, null, null), fullUrl, "MessageHeader", id); } } - private void validateBundleReference(List errors, List entries, Element ref, String name, NodeStack stack, String fullUrl, String type, String id) { + private void validateBundleReference(List errors, Element bundle, List entries, Element ref, String name, NodeStack stack, String fullUrl, String type, String id) { String reference = null; try { reference = ref.getNamedChildValue("reference"); @@ -370,7 +371,7 @@ public class BundleValidator extends BaseValidator { } if (ref != null && !Utilities.noString(reference) && !reference.startsWith("#")) { - Element target = resolveInBundle(entries, reference, fullUrl, type, id); + Element target = resolveInBundle(bundle, entries, reference, fullUrl, type, id); rule(errors, IssueType.INVALID, ref.line(), ref.col(), stack.addToLiteralPath("reference"), target != null, I18nConstants.BUNDLE_BUNDLE_ENTRY_NOTFOUND, reference, name); } @@ -413,7 +414,7 @@ public class BundleValidator extends BaseValidator { for (EntrySummary e : entryList) { Set references = findReferences(e.getEntry()); for (String ref : references) { - Element tgt = resolveInBundle(entries, ref, e.getEntry().getChildValue(FULL_URL), e.getResource().fhirType(), e.getResource().getIdBase()); + Element tgt = resolveInBundle(bundle, entries, ref, e.getEntry().getChildValue(FULL_URL), e.getResource().fhirType(), e.getResource().getIdBase()); if (tgt != null) { EntrySummary t = entryForTarget(entryList, tgt); if (t != null ) { From bdf78cad791770ad61048bb3c24c8fa0e1a98156 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Mon, 23 May 2022 21:25:03 +1000 Subject: [PATCH 2/2] fix total offset error --- .../fhir/validation/instance/InstanceValidator.java | 2 +- .../fhir/validation/instance/PercentageTracker.java | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java index ff10ddba6..94b009123 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java @@ -4397,7 +4397,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat // this method is reentrant, but also the right place to tell the user what is going on if it's the root. // if we're not at the root, we don't report progress pctOwned = true; - pct = new PercentageTracker(resource.countDescendents(), resource.fhirType(), defn.getUrl()); + pct = new PercentageTracker(resource.countDescendents()+1, resource.fhirType(), defn.getUrl()); } if (BUNDLE.equals(element.fhirType())) { if (debug) { diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java index 536ce0d38..2d2cddb70 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/PercentageTracker.java @@ -25,11 +25,14 @@ public class PercentageTracker { if (e.getInstanceId() != instance) { e.setInstanceId(instance); current++; - int pct = (current*100) / total; - if (pct > last + 5) { - while (last + 5 < pct) { + int pct = total == 0 ? 0: (current*100) / total; + if (pct > last + 2) { + while (last + 2 < pct) { System.out.print("."); - last = last + 5; + last = last + 2; + if (last % 20 == 0) { + System.out.print(""+last); + } } } }