From 38cf5e01dcb79d62ad084e3ef10869964bcb5eb4 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Tue, 5 Sep 2023 00:13:24 +1000 Subject: [PATCH 1/3] fix DF markdown engine to escape < properly --- .../main/java/org/hl7/fhir/utilities/MarkDownProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/MarkDownProcessor.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/MarkDownProcessor.java index 6d11a1b69..cddc35c3a 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/MarkDownProcessor.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/MarkDownProcessor.java @@ -42,6 +42,7 @@ import org.commonmark.node.Node; import org.commonmark.parser.Parser; import org.commonmark.renderer.html.HtmlRenderer; +import com.github.rjeschke.txtmark.Configuration; import com.github.rjeschke.txtmark.Processor; public class MarkDownProcessor { @@ -65,7 +66,8 @@ public class MarkDownProcessor { return ""; } switch (dialect) { - case DARING_FIREBALL : return Processor.process(source); + case DARING_FIREBALL : + return Processor.process(source, Configuration.builder().enableSafeMode().enablePanicMode().build()); case COMMON_MARK : return processCommonMark(preProcess(source)); default: throw new Error("Unknown Markdown Dialect: "+dialect.toString()+" at "+context); } From cdda99c290b59ca2e0f65359b3f9df56b641293c Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Tue, 5 Sep 2023 00:14:19 +1000 Subject: [PATCH 2/3] beter xhtml logging for debugging --- .../org/hl7/fhir/utilities/xhtml/XhtmlParser.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/xhtml/XhtmlParser.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/xhtml/XhtmlParser.java index 1514b9d58..a54d3b173 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/xhtml/XhtmlParser.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/xhtml/XhtmlParser.java @@ -47,6 +47,7 @@ import java.util.Set; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.exceptions.FHIRFormatError; import org.hl7.fhir.utilities.StringPair; +import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.i18n.I18nConstants; import org.hl7.fhir.utilities.xhtml.XhtmlNode.Location; @@ -59,6 +60,7 @@ import org.xmlpull.v1.XmlPullParserException; public class XhtmlParser { public static final String XHTML_NS = "http://www.w3.org/1999/xhtml"; private static final char END_OF_CHARS = (char) -1; + private static final boolean DEBUG = false; public class NamespaceNormalizationMap { @@ -1285,7 +1287,14 @@ public class XhtmlParser { public XhtmlNode parseFragment(String source) throws IOException, FHIRException { rdr = new StringReader(source); - return parseFragment(); + try { + return parseFragment(); + } catch (Exception e) { + if (DEBUG) { + TextFile.stringToFile(source, Utilities.path("[tmp]", "html-fail.xhtml")); + } + throw e; + } } public XhtmlNode parseFragment(InputStream input) throws IOException, FHIRException { From beb89b575a7fa7958da7c8399ebbb1b1bb04bc22 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Tue, 5 Sep 2023 23:09:18 +1000 Subject: [PATCH 3/3] rework type checking on target profiles to support imposesProfile --- .../convertors/analytics/PackageVisitor.java | 32 ++++++++++++-- .../conformance/profile/ProfileUtilities.java | 43 ++++++++++++------- .../hl7/fhir/r5/model/ElementDefinition.java | 2 +- .../org/hl7/fhir/utilities/Utilities.java | 4 ++ .../utilities/json/parser/JsonParser.java | 13 ++++++ .../hl7/fhir/utilities/npm/PackageList.java | 17 ++++++++ 6 files changed, 90 insertions(+), 21 deletions(-) diff --git a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/PackageVisitor.java b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/PackageVisitor.java index e11415fd4..d6fd8848a 100644 --- a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/PackageVisitor.java +++ b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/PackageVisitor.java @@ -1,5 +1,8 @@ package org.hl7.fhir.convertors.analytics; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -42,7 +45,8 @@ public class PackageVisitor { private boolean current; private IPackageVisitorProcessor processor; private FilesystemPackageCacheManager pcm; - private PackageClient pc; + private PackageClient pc; + private String cache; public List getResourceTypes() { return resourceTypes; @@ -76,6 +80,14 @@ public class PackageVisitor { + public String getCache() { + return cache; + } + + public void setCache(String cache) { + this.cache = cache; + } + public void setCorePackages(boolean corePackages) { this.corePackages = corePackages; } @@ -109,6 +121,8 @@ public class PackageVisitor { System.out.println("Finding packages"); pc = new PackageClient(PackageServer.primaryServer()); pcm = new FilesystemPackageCacheManager(org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode.USER); + + Set pidList = getAllPackages(); Map cpidMap = getAllCIPackages(); Set cpidSet = new HashSet<>(); @@ -118,7 +132,7 @@ public class PackageVisitor { processCurrentPackage(s, cpidMap.get(s), cpidSet, i, cpidMap.size()); i++; } - Set pidList = getAllPackages(); + System.out.println("Go: "+pidList.size()+" published packages"); i = 0; for (String pid : pidList) { @@ -161,11 +175,21 @@ public class PackageVisitor { private void processCurrentPackage(String url, String pid, Set cpidSet, int i, int t) { try { + long ms1 = System.currentTimeMillis(); String[] p = url.split("\\/"); String repo = "https://build.fhir.org/ig/"+p[0]+"/"+p[1]; - NpmPackage npm = NpmPackage.fromUrl(repo+"/package.tgz"); + JsonObject manifest = JsonParser.parseObjectFromUrl(repo+"/package.manifest.json"); + File co = new File(Utilities.path(cache, pid+"."+manifest.asString("date")+".tgz")); + if (!co.exists()) { + SimpleHTTPClient fetcher = new SimpleHTTPClient(); + HTTPResult res = fetcher.get(repo+"/package.tgz?nocache=" + System.currentTimeMillis()); + res.checkThrowException(); + TextFile.bytesToFile(res.getContent(), co); + } + NpmPackage npm = NpmPackage.fromPackage(new FileInputStream(co)); String fv = npm.fhirVersion(); cpidSet.add(pid); + long ms2 = System.currentTimeMillis(); if (corePackages || !corePackage(npm)) { int c = 0; @@ -182,7 +206,7 @@ public class PackageVisitor { } } } - System.out.println("Processed: "+pid+"#current: "+c+" resources ("+i+" of "+t+")"); + System.out.println("Processed: "+pid+"#current: "+c+" resources ("+i+" of "+t+", "+(ms2-ms1)+"/"+(System.currentTimeMillis()-ms2)+"ms)"); } } catch (Exception e) { System.out.println("Unable to process: "+pid+"#current: "+e.getMessage()); diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/profile/ProfileUtilities.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/profile/ProfileUtilities.java index 2128150d2..8b6cff423 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/profile/ProfileUtilities.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/conformance/profile/ProfileUtilities.java @@ -2839,10 +2839,11 @@ public class ProfileUtilities extends TranslatingUtilities { boolean ok = false; CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); String t = ts.getWorkingCode(); + String tDesc = ts.toString(); for (TypeRefComponent td : base.getType()) {; boolean matchType = false; String tt = td.getWorkingCode(); - b.append(tt); + b.append(td.toString()); if (td.hasCode() && (tt.equals(t))) { matchType = true; } @@ -2869,20 +2870,7 @@ public class ProfileUtilities extends TranslatingUtilities { // check that any derived target has a reference chain back to one of the base target profiles for (UriType u : ts.getTargetProfile()) { String url = u.getValue(); - boolean tgtOk = !td.hasTargetProfile() || td.hasTargetProfile(url); - while (url != null && !tgtOk) { - StructureDefinition sd = context.fetchResource(StructureDefinition.class, url); - if (sd == null) { - if (messages != null) { - messages.add(new ValidationMessage(Source.InstanceValidator, IssueType.BUSINESSRULE, path, "Cannot check whether the target profile " + url + " on "+derived.getPath()+" is valid constraint on the base because it is not known", IssueSeverity.WARNING)); - } - url = null; - tgtOk = true; // suppress error message - } else { - url = sd.getBaseDefinition(); - tgtOk = td.hasTargetProfile(url); - } - } + boolean tgtOk = !td.hasTargetProfile() || sdConformsToTargets(path, derived.getPath(), url, td); if (tgtOk) { ok = true; } else { @@ -2899,11 +2887,34 @@ public class ProfileUtilities extends TranslatingUtilities { } } if (!ok) { - throw new DefinitionException(context.formatMessage(I18nConstants.STRUCTUREDEFINITION__AT__ILLEGAL_CONSTRAINED_TYPE__FROM__IN_, purl, derived.getPath(), t, b.toString(), srcSD.getUrl())); + throw new DefinitionException(context.formatMessage(I18nConstants.STRUCTUREDEFINITION__AT__ILLEGAL_CONSTRAINED_TYPE__FROM__IN_, purl, derived.getPath(), tDesc, b.toString(), srcSD.getUrl())); } } + private boolean sdConformsToTargets(String path, String dPath, String url, TypeRefComponent td) { + if (td.hasTargetProfile(url)) { + return true; + } + StructureDefinition sd = context.fetchResource(StructureDefinition.class, url); + if (sd == null) { + if (messages != null) { + messages.add(new ValidationMessage(Source.InstanceValidator, IssueType.BUSINESSRULE, path, "Cannot check whether the target profile " + url + " on "+dPath+" is valid constraint on the base because it is not known", IssueSeverity.WARNING)); + } + return true; + } else { + if (sd.hasBaseDefinition() && sdConformsToTargets(path, dPath, sd.getBaseDefinition(), td)) { + return true; + } + for (Extension ext : sd.getExtensionsByUrl(ToolingExtensions.EXT_SD_IMPOSE_PROFILE)) { + if (sdConformsToTargets(path, dPath, ext.getValueCanonicalType().asStringValue(), td)) { + return true; + } + } + } + return false; + } + private void checkTypeOk(ElementDefinition dest, String ft, StructureDefinition sd, String fieldName) { boolean ok = false; Set types = new HashSet<>(); diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/ElementDefinition.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/ElementDefinition.java index 4b6a5eb7a..123304c4e 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/ElementDefinition.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/ElementDefinition.java @@ -2587,7 +2587,7 @@ public boolean hasTarget() { res = res + "}"; } if (hasTargetProfile()) { - res = res + "->("; + res = res + "("; boolean first = true; for (CanonicalType s : getTargetProfile()) { if (first) first = false; else res = res + "|"; diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java index 5a77e88de..17029afdd 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java @@ -647,6 +647,10 @@ public class Utilities { return PathBuilder.getPathBuilder().buildPath(args); } + public static File pathFile(String... args) throws IOException { + return new File(PathBuilder.getPathBuilder().buildPath(args)); + } + public static String path(File f, String... args) throws IOException { String[] a = new String[args.length+1]; a[0] = f.getAbsolutePath(); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonParser.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonParser.java index 249e37604..746d14793 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonParser.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonParser.java @@ -1,6 +1,7 @@ package org.hl7.fhir.utilities.json.parser; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -167,6 +168,12 @@ public class JsonParser { compose(element, stream, false); } + public static void compose(JsonElement element, File file) throws IOException { + FileOutputStream fo = new FileOutputStream(file); + compose(element, fo, false); + fo.close(); + } + public static byte[] composeBytes(JsonElement element) { return composeBytes(element, false); } @@ -179,6 +186,12 @@ public class JsonParser { byte[] cnt = composeBytes(element, pretty); stream.write(cnt); } + public static void compose(JsonElement element, File file, boolean pretty) throws IOException { + byte[] cnt = composeBytes(element, pretty); + FileOutputStream fo = new FileOutputStream(file); + fo.write(cnt); + fo.close(); + } public static byte[] composeBytes(JsonElement element, boolean pretty) { String s = compose(element, pretty); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/PackageList.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/PackageList.java index 220e610ef..bc555c712 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/PackageList.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/PackageList.java @@ -5,7 +5,9 @@ import java.io.IOException; import java.text.ParseException; import java.time.Instant; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.hl7.fhir.utilities.FhirPublication; import org.hl7.fhir.utilities.TextFile; @@ -67,6 +69,10 @@ public class PackageList { json.set("date", date); } + public String name() { + return json.asString("name"); + } + public List subPackages() { List list = new ArrayList<>(); if (json.has("sub-packages")) { @@ -169,6 +175,7 @@ public class PackageList { } setDate(date); } + } private String source; @@ -334,5 +341,15 @@ public class PackageList { return json.asString("introduction"); } + public List milestones() { + List list = new ArrayList<>(); + for (PackageListEntry t : versions) { + if (t.name() != null) { + list.add(t); + } + } + return list; + } + }