Merge pull request #637 from hapifhir/gg-202110-CVE-2021-42574

Gg 202110 CVE 2021 42574
This commit is contained in:
Grahame Grieve 2021-11-03 12:08:42 +11:00 committed by GitHub
commit 4cde3369ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 305 additions and 20 deletions

View File

@ -79,7 +79,7 @@ public class VersionConvertorContext<T> {
stack = new Stack<>();
}
stack.push(path);
logger.debug("Pushing path <" + path + "> onto stack. Current path -> " + String.join(",", stack));
// logger.debug("Pushing path <" + path + "> onto stack. Current path -> " + String.join(",", stack));
threadLocalPath.set(stack);
}
@ -96,7 +96,7 @@ public class VersionConvertorContext<T> {
throw new FHIRException("Cannot close path <" + path + ">. Reached unstable state, no stack path available.");
}
String currentPath = stack.pop();
logger.debug("Popping path <" + currentPath + "> off stack. Current path -> " + String.join(",", stack));
// logger.debug("Popping path <" + currentPath + "> off stack. Current path -> " + String.join(",", stack));
if (!path.equals(currentPath)) {
throw new FHIRException("Reached unstable state, current path doesn't match expected path.");
}
@ -127,7 +127,7 @@ public class VersionConvertorContext<T> {
public T getVersionConvertor() {
T result = threadLocalVersionConverter.get();
if (result != null && logger != null) {
logger.debug(result.toString());
// logger.debug(result.toString());
}
return result;
}

View File

@ -36,6 +36,7 @@ public class DicomPackageBuilder {
private String version;
private String dest;
private String source;
private String pattern = "fhir.dicom#{version}.tgz";
public static void main(String[] args) throws FileNotFoundException, ParserConfigurationException, SAXException, IOException {
if (args.length == 0 || args[0].equals("-help")) {
@ -55,12 +56,16 @@ public class DicomPackageBuilder {
DicomPackageBuilder self = new DicomPackageBuilder();
self.setSource(source);
self.setDest(dest);
self.setPattern("{version}/package.tgz");
self.execute();
}
private void execute() throws FileNotFoundException, ParserConfigurationException, SAXException, IOException {
public void execute() throws FileNotFoundException, ParserConfigurationException, SAXException, IOException {
CodeSystem cs = buildCodeSystem();
NPMPackageGenerator gen = new NPMPackageGenerator(Utilities.path(dest, "fhir.dicom#"+version+".tgz"), buildPackage());
String fn = Utilities.path(dest, pattern.replace("{version}", version));
Utilities.createDirectory(Utilities.getDirectoryForFile(fn));
NPMPackageGenerator gen = new NPMPackageGenerator(fn, buildPackage());
int i = 2;
gen.addFile(Category.RESOURCE, "CodeSystem-"+cs.getId()+".json", new JsonParser().setOutputStyle(OutputStyle.NORMAL).composeBytes(cs));
ValueSet vs = buildAllValueSet();
@ -168,15 +173,16 @@ public class DicomPackageBuilder {
return p[0].substring(0, 4)+"."+(1+(p[0].charAt(4) - 'a'))+"."+p[1];
}
private void setVersion(String version) {
this.version = version;
}
private void setDest(String dest) {
public void setDest(String dest) {
this.dest = dest;
}
private void setSource(String source) {
public void setSource(String source) {
this.source = source;
}
public void setPattern(String pattern) {
this.pattern = pattern;
}
}

View File

@ -335,6 +335,7 @@ public class ProfileUtilities extends TranslatingUtilities {
private boolean autoFixSliceNames;
private XVerExtensionManager xver;
private boolean wantFixDifferentialFirstElementType;
private List<String> masterSourceFileNames;
public ProfileUtilities(IWorkerContext context, List<ValidationMessage> messages, ProfileKnowledgeProvider pkp, FHIRPathEngine fpe) {
super();
@ -2452,19 +2453,19 @@ public class ProfileUtilities extends TranslatingUtilities {
if (webUrl != null) {
// also, must touch up the markdown
if (element.hasDefinition())
element.setDefinition(processRelativeUrls(element.getDefinition(), webUrl, baseSpecUrl(), context.getResourceNames()));
element.setDefinition(processRelativeUrls(element.getDefinition(), webUrl, baseSpecUrl(), context.getResourceNames(), masterSourceFileNames));
if (element.hasComment())
element.setComment(processRelativeUrls(element.getComment(), webUrl, baseSpecUrl(), context.getResourceNames()));
element.setComment(processRelativeUrls(element.getComment(), webUrl, baseSpecUrl(), context.getResourceNames(), masterSourceFileNames));
if (element.hasRequirements())
element.setRequirements(processRelativeUrls(element.getRequirements(), webUrl, baseSpecUrl(), context.getResourceNames()));
element.setRequirements(processRelativeUrls(element.getRequirements(), webUrl, baseSpecUrl(), context.getResourceNames(), masterSourceFileNames));
if (element.hasMeaningWhenMissing())
element.setMeaningWhenMissing(processRelativeUrls(element.getMeaningWhenMissing(), webUrl, baseSpecUrl(), context.getResourceNames()));
element.setMeaningWhenMissing(processRelativeUrls(element.getMeaningWhenMissing(), webUrl, baseSpecUrl(), context.getResourceNames(), masterSourceFileNames));
}
}
return element;
}
public static String processRelativeUrls(String markdown, String webUrl, String basePath, List<String> resourceNames) {
public static String processRelativeUrls(String markdown, String webUrl, String basePath, List<String> resourceNames, List<String> filenames) {
StringBuilder b = new StringBuilder();
int i = 0;
while (i < markdown.length()) {
@ -2485,7 +2486,7 @@ public class ProfileUtilities extends TranslatingUtilities {
// This code is trying to guess which relative references are actually to the
// base specification.
//
if (isLikelySourceURLReference(url, resourceNames)) {
if (isLikelySourceURLReference(url, resourceNames, filenames)) {
b.append("](");
b.append(basePath);
i = i + 1;
@ -2507,7 +2508,7 @@ public class ProfileUtilities extends TranslatingUtilities {
}
public static boolean isLikelySourceURLReference(String url, List<String> resourceNames) {
public static boolean isLikelySourceURLReference(String url, List<String> resourceNames, List<String> filenames) {
if (resourceNames != null) {
for (String n : resourceNames) {
if (url.startsWith(n.toLowerCase()+".html")) {
@ -2518,6 +2519,13 @@ public class ProfileUtilities extends TranslatingUtilities {
}
}
}
if (filenames != null) {
for (String n : filenames) {
if (url.startsWith(n.toLowerCase())) {
return true;
}
}
}
return
url.startsWith("extensibility.html") ||
url.startsWith("terminologies.html") ||
@ -6660,5 +6668,14 @@ public class ProfileUtilities extends TranslatingUtilities {
public ElementDefinitionResolution resolveContentRef(StructureDefinition structure, ElementDefinition element) {
return getElementById(structure, structure.getSnapshot().getElement(), element.getContentReference());
}
public List<String> getMasterSourceFileNames() {
return masterSourceFileNames;
}
public void setMasterSourceFileNames(List<String> masterSourceFileNames) {
this.masterSourceFileNames = masterSourceFileNames;
}
}

View File

@ -203,6 +203,9 @@ public interface IResourceValidator {
public boolean isNoExtensibleWarnings();
public IResourceValidator setNoExtensibleWarnings(boolean noExtensibleWarnings);
public boolean isNoUnicodeBiDiControlChars();
public void setNoUnicodeBiDiControlChars(boolean noUnicodeBiDiControlChars);
/**
* Whether being unable to resolve a profile in found in Resource.meta.profile or ElementDefinition.type.profile or targetProfile is an error or just a warning
* @return

View File

@ -0,0 +1,189 @@
package org.hl7.fhir.utilities;
import java.util.List;
import java.util.ArrayList;
public class UnicodeUtilities {
public static class StateStackEntry {
private char c;
private int i;
public StateStackEntry(char c, int i) {
this.c = c;
this.i = i;
}
}
public static class StateStack {
private List<StateStackEntry> list = new ArrayList<>();
public void clear() {
list.clear();
}
public void push(char c, int i) {
list.add(new StateStackEntry(c, i));
}
public void popJustOne(CharSet oneSet) {
if (!list.isEmpty() && oneSet.contains(list.get(list.size()-1).c)) {
list.remove(list.size()-1);
}
}
public void popOneAndOthers(CharSet oneSet, CharSet otherSet) {
boolean found = false;
for (StateStackEntry t : list) {
if (oneSet.contains(t.c)) {
found = true;
break;
}
}
if (found) {
while (!list.isEmpty() && (oneSet.contains(list.get(list.size()-1).c) || otherSet.contains(list.get(list.size()-1).c))) {
boolean done = oneSet.contains(list.get(list.size()-1).c);
list.remove(list.size()-1);
if (done) {
break;
}
}
}
}
public boolean empty() {
return list.isEmpty();
}
public String summary() {
return "Unicode Character "+describe(list.get(list.size()-1).c)+" at index "+list.get(list.size()-1).i+" has no terminating match";
}
}
public static class CharSet {
private char[] chars;
public CharSet(char... chars) {
this.chars = chars;
}
public boolean contains(char c) {
for (char t : chars) {
if (c == t) {
return true;
}
}
return false;
}
}
public static final char LRE = '\u202a';
public static final char RLE = '\u202b';
public static final char PDF = '\u202c';
public static final char LRO = '\u202d';
public static final char RLO = '\u202e';
public static final char LRI = '\u2066';
public static final char RLI = '\u2067';
public static final char FSI = '\u2068';
public static final char PDI = '\u2069';
public static final char LRM = '\u200E';
public static final char RLM = '\u200F';
public static final char ALM = '\u061C';
public static final char PARA = '\n';
private static CharSet allBiDiChars = new CharSet(LRE, RLE, PDF, LRO, RLO, LRI, RLI, FSI, PDI, LRM, RLM, ALM, PARA);
public static boolean hasBiDiChars(String src) {
for (char c : src.toCharArray()) {
if (allBiDiChars.contains(c)) {
return true;
}
}
return false;
}
/**
* returns null if src is well formed, or a description of a structure problem with bi-directional characters
* @param src
* @return
*/
public static String checkUnicodeWellFormed(String src) {
StateStack ss = new StateStack();
for (int i = 0; i < src.length(); i++) {
char c = src.charAt(i);
if (allBiDiChars.contains(c)) {
switch (c) {
case PARA:
ss.clear();
break;
case LRO:
case RLO:
ss.push(c, i);
break;
case PDF:
ss.popJustOne(new CharSet(LRE, RLE, LRO, RLO, LRM, RLM, ALM));
break;
case LRI:
case RLI:
case FSI:
ss.push(c, i);
break;
case PDI:
ss.popOneAndOthers(new CharSet(LRI, RLI, FSI), new CharSet(LRE, RLE, LRO, RLO, LRM, RLM, ALM));
break;
case LRM:
case RLM:
case ALM:
ss.push(c, i);
break;
}
}
}
if (ss.empty()) {
return null;
} else {
return ss.summary();
}
}
public static String describe(char c) {
switch (c) {
case LRE: return "LRE";
case RLE: return "RLE";
case PDF: return "PDF";
case LRO: return "LRO";
case RLO: return "RLO";
case LRI: return "LRI";
case RLI: return "RLI";
case FSI: return "FSI";
case PDI: return "PDI";
case LRM: return "LRM";
case RLM: return "RLM";
case ALM: return "ALM";
case PARA: return "PARA";
}
return String.valueOf(c);
}
public static Object replaceBiDiChars(String s) {
if (s == null) {
return null;
}
StringBuilder b = new StringBuilder();
for (char c : s.toCharArray()) {
if (allBiDiChars.contains(c)) {
b.append("|"+describe(c)+"|");
} else {
b.append(c);
}
}
return b.toString();
}
}

View File

@ -649,6 +649,8 @@ public class I18nConstants {
public static final String BUNDLE_SEARCH_ENTRY_WRONG_RESOURCE_TYPE_NO_MODE = "BUNDLE_SEARCH_ENTRY_WRONG_RESOURCE_TYPE_NO_MODE";
public static final String BUNDLE_SEARCH_NO_MODE = "BUNDLE_SEARCH_NO_MODE";
public static final String BUNDLE_SEARCH_ENTRY_WRONG_RESOURCE_TYPE_OUTCOME = "BUNDLE_SEARCH_ENTRY_WRONG_RESOURCE_TYPE_OUTCOME";
public static final String UNICODE_BIDI_CONTROLS_CHARS_DISALLOWED = "UNICODE_BIDI_CONTROLS_CHARS_DISALLOWED";
public static final String UNICODE_BIDI_CONTROLS_CHARS_MATCH = "UNICODE_BIDI_CONTROLS_CHARS_MATCH";
}

View File

@ -660,3 +660,5 @@ BUNDLE_SEARCH_NO_MODE = SearchSet bundles should have search modes on the entrie
INV_FAILED = Rule {0} Failed
PATTERN_CHECK_STRING = The pattern [{0}] defined in the profile {1} not found. Issues: {2}
TYPE_SPECIFIC_CHECKS_DT_URL_EXAMPLE = Example URLs are not allowed in this context ({0})
UNICODE_BIDI_CONTROLS_CHARS_DISALLOWED = The Unicode sequence has bi-di control characters which are not allowed in this context: {1}
UNICODE_BIDI_CONTROLS_CHARS_MATCH = The Unicode sequence has unterminated bi-di control characters (see CVE-2021-42574): {1}

View File

@ -0,0 +1,28 @@
package org.hl7.fhir.utilities;
import java.io.IOException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
public class UnicodeUtilitiesTests {
@Test
@DisplayName("Test Unicode bidi-detection working")
public void testUnicodeBiDiDetection() throws IOException {
Assertions.assertFalse(UnicodeUtilities.hasBiDiChars("abc"));
Assertions.assertTrue(UnicodeUtilities.hasBiDiChars(UnicodeUtilities.RLI + "abc" + UnicodeUtilities.PDI));
}
@Test
@DisplayName("Test Unicode match checking working")
public void testUnicodeMatchChecking() throws IOException {
Assertions.assertNull(UnicodeUtilities.checkUnicodeWellFormed("abc"));
Assertions.assertNull(UnicodeUtilities.checkUnicodeWellFormed(UnicodeUtilities.RLI + "abc" + UnicodeUtilities.PDI));
Assertions.assertNull(UnicodeUtilities.checkUnicodeWellFormed(UnicodeUtilities.RLI + " "+ UnicodeUtilities.LRI + "a b c "+
UnicodeUtilities.PDI+" "+UnicodeUtilities.LRI+" d e f "+UnicodeUtilities.PDI+" "+UnicodeUtilities.PDI));
Assertions.assertEquals(UnicodeUtilities.checkUnicodeWellFormed("'''subject funds from back account then "+UnicodeUtilities.RLI + "''' ;return"),
"Unicode Character RLI at index 40 has no terminating match");
}
}

View File

@ -149,6 +149,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IPackageInst
@Getter @Setter private ICanonicalResourceLocator locator;
@Getter @Setter private boolean assumeValidRestReferences;
@Getter @Setter private boolean noExtensibleBindingMessages;
@Getter @Setter private boolean noUnicodeBiDiControlChars;
@Getter @Setter private boolean securityChecks;
@Getter @Setter private boolean crumbTrails;
@Getter @Setter private boolean allowExampleUrls;
@ -518,6 +519,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IPackageInst
validator.getBundleValidationRules().addAll(bundleValidationRules);
validator.getValidationControl().putAll(validationControl);
validator.setQuestionnaireMode(questionnaireMode);
validator.setNoUnicodeBiDiControlChars(noUnicodeBiDiControlChars);
if (format == FhirFormat.SHC) {
igLoader.loadIg(getIgs(), getBinaries(), SHCParser.CURRENT_PACKAGE, true);
}

View File

@ -38,6 +38,8 @@ public class CliContext {
private boolean noInternalCaching = false; // internal, for when debugging terminology validation
@JsonProperty("noExtensibleBindingMessages")
private boolean noExtensibleBindingMessages = false;
@JsonProperty("noUnicodeBiDiControlChars")
private boolean noUnicodeBiDiControlChars = false;
@JsonProperty("noInvariants")
private boolean noInvariants = false;
@JsonProperty("wantInvariantsInMessages")
@ -528,6 +530,14 @@ public class CliContext {
this.outputStyle = outputStyle;
}
public boolean isNoUnicodeBiDiControlChars() {
return noUnicodeBiDiControlChars;
}
public void setNoUnicodeBiDiControlChars(boolean noUnicodeBiDiControlChars) {
this.noUnicodeBiDiControlChars = noUnicodeBiDiControlChars;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
@ -542,6 +552,7 @@ public class CliContext {
canDoNative == that.canDoNative &&
noInternalCaching == that.noInternalCaching &&
noExtensibleBindingMessages == that.noExtensibleBindingMessages &&
noUnicodeBiDiControlChars == that.noUnicodeBiDiControlChars &&
noInvariants == that.noInvariants &&
wantInvariantsInMessages == that.wantInvariantsInMessages &&
Objects.equals(map, that.map) &&
@ -570,7 +581,9 @@ public class CliContext {
@Override
public int hashCode() {
return Objects.hash(doNative, anyExtensionsAllowed, hintAboutNonMustSupport, recursive, doDebug, assumeValidRestReferences, canDoNative, noInternalCaching, noExtensibleBindingMessages, noInvariants, wantInvariantsInMessages, map, output, htmlOutput, txServer, sv, txLog, mapLog, lang, fhirpath, snomedCT, targetVer, igs, questionnaireMode, profiles, sources, mode, locale, locations, crumbTrails, showTimes, allowExampleUrls, outputStyle);
return Objects.hash(doNative, anyExtensionsAllowed, hintAboutNonMustSupport, recursive, doDebug, assumeValidRestReferences, canDoNative, noInternalCaching,
noExtensibleBindingMessages, noInvariants, wantInvariantsInMessages, map, output, htmlOutput, txServer, sv, txLog, mapLog, lang, fhirpath, snomedCT,
targetVer, igs, questionnaireMode, profiles, sources, mode, locale, locations, crumbTrails, showTimes, allowExampleUrls, outputStyle, noUnicodeBiDiControlChars);
}
@Override
@ -585,6 +598,7 @@ public class CliContext {
", canDoNative=" + canDoNative +
", noInternalCaching=" + noInternalCaching +
", noExtensibleBindingMessages=" + noExtensibleBindingMessages +
", noUnicodeBiDiControlChars=" + noUnicodeBiDiControlChars +
", noInvariants=" + noInvariants +
", wantInvariantsInMessages=" + wantInvariantsInMessages +
", map='" + map + '\'' +

View File

@ -290,6 +290,7 @@ public class ValidationService {
validator.setAssumeValidRestReferences(cliContext.isAssumeValidRestReferences());
validator.setShowMessagesFromReferences(cliContext.isShowMessagesFromReferences());
validator.setNoExtensibleBindingMessages(cliContext.isNoExtensibleBindingMessages());
validator.setNoUnicodeBiDiControlChars(cliContext.isNoUnicodeBiDiControlChars());
validator.setNoInvariantChecks(cliContext.isNoInvariants());
validator.setWantInvariantInMessage(cliContext.isWantInvariantsInMessages());
validator.setSecurityChecks(cliContext.isSecurityChecks());

View File

@ -54,6 +54,7 @@ public class Params {
public static final String RIGHT = "-right";
public static final String NO_INTERNAL_CACHING = "-no-internal-caching";
public static final String NO_EXTENSIBLE_BINDING_WARNINGS = "-no-extensible-binding-warnings";
public static final String NO_UNICODE_BIDI_CONTROL_CHARS = "-no_unicode_bidi_control_chars";
public static final String NO_INVARIANTS = "-no-invariants";
public static final String WANT_INVARIANTS_IN_MESSAGES = "-want-invariants-in-messages";
public static final String SECURITY_CHECKS = "-security-checks";
@ -165,6 +166,8 @@ public class Params {
cliContext.setNoInternalCaching(true);
} else if (args[i].equals(NO_EXTENSIBLE_BINDING_WARNINGS)) {
cliContext.setNoExtensibleBindingMessages(true);
} else if (args[i].equals(NO_UNICODE_BIDI_CONTROL_CHARS)) {
cliContext.setNoUnicodeBiDiControlChars(true);
} else if (args[i].equals(NO_INVARIANTS)) {
cliContext.setNoInvariants(true);
} else if (args[i].equals(WANT_INVARIANTS_IN_MESSAGES)) {

View File

@ -140,6 +140,7 @@ import org.hl7.fhir.r5.utils.ToolingExtensions;
import org.hl7.fhir.r5.utils.XVerExtensionManager;
import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
import org.hl7.fhir.utilities.SIDUtilities;
import org.hl7.fhir.utilities.UnicodeUtilities;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.Utilities.DecimalStatus;
import org.hl7.fhir.utilities.VersionUtilities;
@ -363,6 +364,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat
private boolean baseOnly;
private boolean noCheckAggregation;
private boolean wantCheckSnapshotUnchanged;
private boolean noUnicodeBiDiControlChars;
private List<ImplementationGuide> igs = new ArrayList<>();
private List<String> extensionDomains = new ArrayList<String>();
@ -1997,6 +1999,14 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat
rule(errors, IssueType.CODEINVALID, e.line(), e.col(), path, context.getBinding().getStrength() != BindingStrength.REQUIRED, I18nConstants.Terminology_TX_Code_ValueSet_MISSING);
}
return;
} else {
boolean hasBiDiControls = UnicodeUtilities.hasBiDiChars(e.primitiveValue());
if (hasBiDiControls) {
if (rule(errors, IssueType.CODEINVALID, e.line(), e.col(), path, !noUnicodeBiDiControlChars, I18nConstants.UNICODE_BIDI_CONTROLS_CHARS_DISALLOWED, UnicodeUtilities.replaceBiDiChars(e.primitiveValue()))) {
String msg = UnicodeUtilities.checkUnicodeWellFormed(e.primitiveValue());
warning(errors, IssueType.INVALID, e.line(), e.col(), path, msg == null, I18nConstants.UNICODE_BIDI_CONTROLS_CHARS_MATCH, msg);
}
}
}
String regex = context.getExtensionString(ToolingExtensions.EXT_REGEX);
if (regex != null) {
@ -5590,4 +5600,12 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat
this.baseOptions = baseOptions;
}
public boolean isNoUnicodeBiDiControlChars() {
return noUnicodeBiDiControlChars;
}
public void setNoUnicodeBiDiControlChars(boolean noUnicodeBiDiControlChars) {
this.noUnicodeBiDiControlChars = noUnicodeBiDiControlChars;
}
}

View File

@ -19,7 +19,7 @@
<properties>
<hapi_fhir_version>5.1.0</hapi_fhir_version>
<validator_test_case_version>1.1.71</validator_test_case_version>
<validator_test_case_version>1.1.72-SNAPSHOT</validator_test_case_version>
<junit_jupiter_version>5.7.1</junit_jupiter_version>
<junit_platform_launcher_version>1.7.1</junit_platform_launcher_version>
<maven_surefire_version>3.0.0-M4</maven_surefire_version>