honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add support for Liquid assign + Fix JSON unicode encoding and add character check in validator for illegal XML Unicode characters

This commit is contained in:
Grahame Grieve 2023-07-26 12:32:51 +10:00
parent 7302f52879
commit a3c32d86a0
6 changed files with 177 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/utils/LiquidEngine.java
View File

@ -47,6 +47,7 @@ import org.hl7.fhir.r5.model.ValueSet;
import org.hl7.fhir.r5.utils.FHIRPathEngine.ExpressionNodeWithOffset;
import org.hl7.fhir.r5.utils.FHIRPathEngine.IEvaluationContext;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.i18n.I18nConstants;
import org.hl7.fhir.utilities.xhtml.NodeType;
import org.hl7.fhir.utilities.xhtml.XhtmlNode;
@ -67,17 +68,27 @@ public class LiquidEngine implements IEvaluationContext {
private class LiquidEngineContext {
private Object externalContext;
private Map<String, Base> vars = new HashMap<>();
private Map<String, Base> loopVars = new HashMap<>();
private Map<String, Base> globalVars = new HashMap<>();
public LiquidEngineContext(Object externalContext) {
super();
this.externalContext = externalContext;
globalVars = new HashMap<>();
}
public LiquidEngineContext(Object externalContext, LiquidEngineContext existing) {
super();
this.externalContext = externalContext;
loopVars.putAll(existing.loopVars);
globalVars = existing.globalVars;
}
public LiquidEngineContext(LiquidEngineContext existing) {
super();
externalContext = existing.externalContext;
vars.putAll(existing.vars);
loopVars.putAll(existing.loopVars);
globalVars = existing.globalVars;
}
}
@ -185,7 +196,7 @@ public class LiquidEngine implements IEvaluationContext {
String f = lexer.getCurrent();
LiquidFilter filter = LiquidFilter.fromCode(f);
if (filter == null) {
lexer.error("Unknown Liquid filter '"+f+"'");
lexer.error(engine.getWorker().formatMessage(I18nConstants.LIQUID_UNKNOWN_FILTER, f));
}
lexer.next();
if (!lexer.done() && lexer.getCurrent().equals(":")) {
@ -195,7 +206,7 @@ public class LiquidEngine implements IEvaluationContext {
compiled.add(new LiquidExpressionNode(filter, null));
}
} else {
lexer.error("Unexpected syntax parsing liquid statement");
lexer.error(engine.getWorker().formatMessage(I18nConstants.LIQUID_UNKNOWN_SYNTAX));
}
}
}
@ -305,6 +316,30 @@ public class LiquidEngine implements IEvaluationContext {
}
}
private class LiquidAssign extends LiquidNode {
private String varName;
private String expression;
private ExpressionNode compiled;
@Override
public void evaluate(StringBuilder b, Base resource, LiquidEngineContext ctxt) throws FHIRException {
if (compiled == null) {
boolean dbl = engine.isAllowDoubleQuotes();
engine.setAllowDoubleQuotes(true);
ExpressionNodeWithOffset po = engine.parsePartial(expression, 0);
compiled = po.getNode();
engine.setAllowDoubleQuotes(dbl);
}
List<Base> list = engine.evaluate(ctxt, resource, resource, resource, compiled);
if (list.isEmpty()) {
ctxt.globalVars.remove(varName);
} else if (list.size() == 1) {
ctxt.globalVars.put(varName, list.get(0));
} else {
throw new Error("Assign returned a list?");
}
}
}
private class LiquidFor extends LiquidNode {
private String varName;
private String condition;
@ -343,7 +378,10 @@ public class LiquidEngine implements IEvaluationContext {
if (limit >= 0 && i == limit) {
break;
}
lctxt.vars.put(varName, o);
if (lctxt.globalVars.containsKey(varName)) {
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_VARIABLE_ALREADY_ASSIGNED, varName));
}
lctxt.loopVars.put(varName, o);
boolean wantBreak = false;
for (LiquidNode n : body) {
try {
@ -372,7 +410,7 @@ public class LiquidEngine implements IEvaluationContext {
} else if (cnt.startsWith("limit")) {
cnt = cnt.substring(5).trim();
if (!cnt.startsWith(":")) {
throw new FHIRException("Exception evaluating "+src+": limit is not followed by ':'");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_COLON, src));
}
cnt = cnt.substring(1).trim();
int i = 0;
@ -380,14 +418,14 @@ public class LiquidEngine implements IEvaluationContext {
i++;
}
if (i == 0) {
throw new FHIRException("Exception evaluating "+src+": limit is not followed by a number");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_NUMBER, src));
}
limit = Integer.parseInt(cnt.substring(0, i));
cnt = cnt.substring(i);
} else if (cnt.startsWith("offset")) {
cnt = cnt.substring(6).trim();
if (!cnt.startsWith(":")) {
throw new FHIRException("Exception evaluating "+src+": limit is not followed by ':'");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_COLON, src));
}
cnt = cnt.substring(1).trim();
int i = 0;
@ -395,12 +433,12 @@ public class LiquidEngine implements IEvaluationContext {
i++;
}
if (i == 0) {
throw new FHIRException("Exception evaluating "+src+": limit is not followed by a number");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_NUMBER, src));
}
offset = Integer.parseInt(cnt.substring(0, i));
cnt = cnt.substring(i);
} else {
throw new FHIRException("Exception evaluating "+src+": unexpected content at "+cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_UNEXPECTED, cnt));
}
}
}
@ -415,9 +453,9 @@ public class LiquidEngine implements IEvaluationContext {
String src = includeResolver.fetchInclude(LiquidEngine.this, page);
LiquidParser parser = new LiquidParser(src);
LiquidDocument doc = parser.parse(page);
LiquidEngineContext nctxt = new LiquidEngineContext(ctxt.externalContext);
LiquidEngineContext nctxt = new LiquidEngineContext(ctxt.externalContext, ctxt);
Tuple incl = new Tuple();
nctxt.vars.put("include", incl);
nctxt.loopVars.put("include", incl);
for (String s : params.keySet()) {
incl.addProperty(s, engine.evaluate(ctxt, resource, resource, resource, params.get(s)));
}
@ -474,11 +512,11 @@ public class LiquidEngine implements IEvaluationContext {
cnt = "," + cnt.substring(5).trim();
while (!Utilities.noString(cnt)) {
if (!cnt.startsWith(",")) {
throw new FHIRException("Script " + name + ": Script " + name + ": Found " + cnt.charAt(0) + " expecting ',' parsing cycle");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_EXPECTING, name, cnt.charAt(0), ','));
}
cnt = cnt.substring(1).trim();
if (!cnt.startsWith("\"")) {
throw new FHIRException("Script " + name + ": Script " + name + ": Found " + cnt.charAt(0) + " expecting '\"' parsing cycle");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_EXPECTING, name, cnt.charAt(0), '"'));
}
cnt = cnt.substring(1);
int i = 0;
@ -486,7 +524,7 @@ public class LiquidEngine implements IEvaluationContext {
i++;
}
if (i == cnt.length()) {
throw new FHIRException("Script " + name + ": Script " + name + ": Found unterminated string parsing cycle");
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_UNTERMINATED, name));
}
res.list.add(cnt.substring(0, i));
cnt = cnt.substring(i + 1).trim();
@ -518,8 +556,10 @@ public class LiquidEngine implements IEvaluationContext {
list.add(parseCycle(cnt));
else if (cnt.startsWith("include "))
list.add(parseInclude(cnt.substring(7).trim()));
else if (cnt.startsWith("assign "))
list.add(parseAssign(cnt.substring(6).trim()));
else
throw new FHIRException("Script " + name + ": Script " + name + ": Unknown flow control statement " + cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_UNKNOWN_FLOW_STMT,name, cnt));
} else { // next2() == '{'
list.add(parseStatement());
}
@ -533,7 +573,7 @@ public class LiquidEngine implements IEvaluationContext {
n.closeUp();
if (terminators.length > 0)
if (!isTerminator(close, terminators))
throw new FHIRException("Script " + name + ": Script " + name + ": Found end of script looking for " + terminators);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_UNKNOWN_NOEND, name, terminators));
return close;
}
@ -577,7 +617,7 @@ public class LiquidEngine implements IEvaluationContext {
while (i < cnt.length() && !Character.isWhitespace(cnt.charAt(i)))
i++;
if (i == cnt.length() || i == 0)
throw new FHIRException("Script " + name + ": Error reading include: " + cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_INCLUDE, name + ": Error reading include: " + cnt));
LiquidInclude res = new LiquidInclude();
res.page = cnt.substring(0, i);
while (i < cnt.length() && Character.isWhitespace(cnt.charAt(i)))
@ -587,10 +627,10 @@ public class LiquidEngine implements IEvaluationContext {
while (i < cnt.length() && cnt.charAt(i) != '=')
i++;
if (i >= cnt.length() || j == i)
throw new FHIRException("Script " + name + ": Error reading include: " + cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_INCLUDE, name, cnt));
String n = cnt.substring(j, i);
if (res.params.containsKey(n))
throw new FHIRException("Script " + name + ": Error reading include: " + cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_INCLUDE, name, cnt));
i++;
ExpressionNodeWithOffset t = engine.parsePartial(cnt, i);
i = t.getOffset();
@ -607,13 +647,16 @@ public class LiquidEngine implements IEvaluationContext {
i++;
LiquidFor res = new LiquidFor();
res.varName = cnt.substring(0, i);
if ("include".equals(res.varName)) {
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_VARIABLE_ILLEGAL, res.varName));
}
while (Character.isWhitespace(cnt.charAt(i)))
i++;
int j = i;
while (!Character.isWhitespace(cnt.charAt(i)))
i++;
if (!"in".equals(cnt.substring(j, i)))
throw new FHIRException("Script " + name + ": Script " + name + ": Error reading loop: " + cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_LOOP, name, cnt));
res.condition = cnt.substring(i).trim();
parseList(res.body, false, new String[] { "endloop" });
return res;
@ -625,13 +668,16 @@ public class LiquidEngine implements IEvaluationContext {
i++;
LiquidFor res = new LiquidFor();
res.varName = cnt.substring(0, i);
if ("include".equals(res.varName)) {
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_VARIABLE_ILLEGAL, res.varName));
}
while (Character.isWhitespace(cnt.charAt(i)))
i++;
int j = i;
while (!Character.isWhitespace(cnt.charAt(i)))
i++;
if (!"in".equals(cnt.substring(j, i)))
throw new FHIRException("Script " + name + ": Script " + name + ": Error reading loop: " + cnt);
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_LOOP, name, cnt));
res.condition = cnt.substring(i).trim();
String term = parseList(res.body, true, new String[] { "endfor", "else" });
if ("else".equals(term)) {
@ -640,6 +686,20 @@ public class LiquidEngine implements IEvaluationContext {
return res;
}
private LiquidNode parseAssign(String cnt) throws FHIRException {
int i = 0;
while (!Character.isWhitespace(cnt.charAt(i)))
i++;
LiquidAssign res = new LiquidAssign();
res.varName = cnt.substring(0, i);
while (Character.isWhitespace(cnt.charAt(i)))
i++;
int j = i;
while (!Character.isWhitespace(cnt.charAt(i)))
i++;
res.expression = cnt.substring(i).trim();
return res;
}
private String parseTag(char ch) throws FHIRException {
grab();
@ -649,7 +709,7 @@ public class LiquidEngine implements IEvaluationContext {
b.append(grab());
}
if (!(next1() == '%' && next2() == '}'))
throw new FHIRException("Script " + name + ": Unterminated Liquid statement {% " + b.toString());
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_NOTERM, name, "{% " + b.toString()));
grab();
grab();
return b.toString().trim();
@ -663,7 +723,7 @@ public class LiquidEngine implements IEvaluationContext {
b.append(grab());
}
if (!(next1() == '}' && next2() == '}'))
throw new FHIRException("Script " + name + ": Unterminated Liquid statement {{ " + b.toString());
throw new FHIRException(engine.getWorker().formatMessage(I18nConstants.LIQUID_SYNTAX_NOTERM, name, "{{ " + b.toString()));
grab();
grab();
LiquidStatement res = new LiquidStatement();
@ -676,8 +736,10 @@ public class LiquidEngine implements IEvaluationContext {
@Override
public List<Base> resolveConstant(Object appContext, String name, boolean beforeContext) throws PathEngineException {
LiquidEngineContext ctxt = (LiquidEngineContext) appContext;
if (ctxt.vars.containsKey(name))
return new ArrayList<Base>(Arrays.asList(ctxt.vars.get(name)));
if (ctxt.loopVars.containsKey(name))
return new ArrayList<Base>(Arrays.asList(ctxt.loopVars.get(name)));
if (ctxt.globalVars.containsKey(name))
return new ArrayList<Base>(Arrays.asList(ctxt.globalVars.get(name)));
if (externalHostServices == null)
return new ArrayList<Base>();
return externalHostServices.resolveConstant(ctxt.externalContext, name, beforeContext);

44
org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/formats/UnicodeCharacterTests.java Normal file
View File

@ -0,0 +1,44 @@
package org.hl7.fhir.r5.formats;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.r5.formats.IParser.OutputStyle;
import org.hl7.fhir.r5.model.Parameters;
import org.hl7.fhir.r5.test.utils.TestingUtilities;
import org.hl7.fhir.utilities.Utilities;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class UnicodeCharacterTests {
@Test
public void testUnicodeXml() throws FHIRFormatError, IOException {
XmlParser xml = new XmlParser();
xml.setOutputStyle(OutputStyle.PRETTY);
Parameters p = (Parameters) xml.parse(TestingUtilities.loadTestResource("r5", "unicode-problem.xml"));
Assertions.assertEquals("invalid: \u0013, not invalid: \r", p.getParameterFirstRep().getValue().primitiveValue());
FileOutputStream o = new FileOutputStream(Utilities.path("[tmp]", "unicode-problem.xml"));
xml.compose(o, p);
o.close();
p = (Parameters) xml.parse(new FileInputStream(Utilities.path("[tmp]", "unicode-problem.xml")));
Assertions.assertEquals("invalid: \u0013, not invalid: \r", p.getParameterFirstRep().getValue().primitiveValue());
}
@Test
public void testUnicodeJson() throws FHIRFormatError, IOException {
JsonParser json = new JsonParser();
json.setOutputStyle(OutputStyle.PRETTY);
Parameters p = (Parameters) json.parse(TestingUtilities.loadTestResource("r5", "unicode-problem.json"));
Assertions.assertEquals("invalid: \u0013, not invalid: \r", p.getParameterFirstRep().getValue().primitiveValue());
FileOutputStream o = new FileOutputStream(Utilities.path("[tmp]", "unicode-problem.json"));
json.compose(o, p);
o.close();
p = (Parameters) json.parse(new FileInputStream(Utilities.path("[tmp]", "unicode-problem.json")));
Assertions.assertEquals("invalid: \u0013, not invalid: \r", p.getParameterFirstRep().getValue().primitiveValue());
}
}

2
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java
View File

@ -984,7 +984,7 @@ public class Utilities {
else if (isWhitespace(c)) {
b.append("\\u"+Utilities.padLeft(Integer.toHexString(c), '0', 4));
} else if (((int) c) < 32)
b.append("\\u" + Utilities.padLeft(String.valueOf((int) c), '0', 4));
b.append("\\u" + Utilities.padLeft(Integer.toHexString(c), '0', 4));
else
b.append(c);
}

18
org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/I18nConstants.java
View File

@ -926,6 +926,24 @@ public class I18nConstants {
public static final String VALUESET_CONCEPT_DISPLAY_PRESENCE_MIXED = "VALUESET_CONCEPT_DISPLAY_PRESENCE_MIXED";
public static final String VALUESET_CONCEPT_DISPLAY_SCT_TAG_MIXED = "VALUESET_CONCEPT_DISPLAY_SCT_TAG_MIXED";
public static final String CS_SCT_IPS_NOT_IPS = "CS_SCT_IPS_NOT_IPS";
public static final String UNICODE_XML_BAD_CHARS = "UNICODE_XML_BAD_CHARS";
public static final String LIQUID_UNKNOWN_FILTER = "LIQUID_UNKNOWN_FILTER";
public static final String LIQUID_UNKNOWN_SYNTAX = "LIQUID_UNKNOWN_SYNTAX";
public static final String LIQUID_SYNTAX_EXPECTING = "LIQUID_SYNTAX_EXPECTING";
public static final String LIQUID_SYNTAX_UNTERMINATED = "LIQUID_SYNTAX_UNTERMINATED";
public static final String LIQUID_UNKNOWN_FLOW_STMT = "LIQUID_UNKNOWN_FLOW_STMT";
public static final String LIQUID_UNKNOWN_NOEND = "LIQUID_UNKNOWN_NOEND";
public static final String LIQUID_SYNTAX_INCLUDE = "LIQUID_SYNTAX_INCLUDE";
public static final String LIQUID_SYNTAX_LOOP = "LIQUID_SYNTAX_LOOP";
public static final String LIQUID_SYNTAX_NOTERM = "LIQUID_SYNTAX_NOTERM";
public static final String LIQUID_UNKNOWN_NOTERM = "LIQUID_UNKNOWN_NOTERM";
public static final String LIQUID_SYNTAX_COLON = "LIQUID_SYNTAX_COLON";
public static final String LIQUID_SYNTAX_NUMBER = "LIQUID_SYNTAX_NUMBER";
public static final String LIQUID_SYNTAX_UNEXPECTED = "LIQUID_SYNTAX_UNEXPECTED";
public static final String LIQUID_VARIABLE_ALREADY_ASSIGNED = "LIQUID_VARIABLE_ALREADY_ASSIGNED";
public static final String LIQUID_VARIABLE_ILLEGAL = "LIQUID_VARIABLE_ILLEGAL";
}

18
org.hl7.fhir.utilities/src/main/resources/Messages.properties
View File

@ -981,3 +981,21 @@ SD_ED_TYPE_WRONG_TYPE_other = The element has a type {0} which is not in the typ
VALUESET_CONCEPT_DISPLAY_PRESENCE_MIXED = This include has some concepts with displays and some without - check that this is what is intended
VALUESET_CONCEPT_DISPLAY_SCT_TAG_MIXED = This SNOMED-CT based include has some concepts with semantic tags (FSN terms) and some without (preferred terms) - check that this is what is intended
CS_SCT_IPS_NOT_IPS = The Snomed CT code {0} ({1}) is not a member of the IPS free set
UNICODE_XML_BAD_CHARS_one = This content includes the character {1} (hex value). This character is illegal in the XML version of FHIR, and there is generally no valid use for such characters
UNICODE_XML_BAD_CHARS_other = This content includes the characters {1} (hex values). These characters are illegal in the XML version of FHIR, and there is generally no valid use for such characters
LIQUID_UNKNOWN_FILTER = Unknown Liquid filter '''{0}''
LIQUID_UNKNOWN_SYNTAX = Unexpected syntax parsing liquid statement
LIQUID_SYNTAX_EXPECTING = Script {0}: Found ''{1}'' expecting ''{2}'' parsing cycle
LIQUID_SYNTAX_UNTERMINATED = Script {0}: Found unterminated string parsing cycle
LIQUID_UNKNOWN_FLOW_STMT = Script {0}: Unknown flow control statement ''{1}''
LIQUID_UNKNOWN_NOEND = Script {0}: Found end of script looking for {1}
LIQUID_SYNTAX_INCLUDE, = Script {0}: Error reading include: {1}
LIQUID_SYNTAX_LOOP = Script {0}: Error reading loop: {1}
LIQUID_SYNTAX_NOTERM = Script {0}: Unterminated Liquid statement {1}
LIQUID_UNKNOWN_NOTERM = Script {0}: Unterminated Liquid statement {1}
LIQUID_SYNTAX_COLON = Exception evaluating {0}: limit is not followed by '':''
LIQUID_SYNTAX_NUMBER = Exception evaluating {0}: limit is not followed by a number
LIQUID_SYNTAX_UNEXPECTED = Exception evaluating {0}: unexpected content at {1}
LIQUID_VARIABLE_ALREADY_ASSIGNED = "Liquid Exception: The variable ''{0}'' already has an assigned value
LIQUID_VARIABLE_ILLEGAL = "Liquid Exception: The variable name ''{0}'' cannot be used

8
org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java
View File

@ -2379,6 +2379,14 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat
ok = false;
}
}
Set<String> badChars = new HashSet<>();
for (char ch : e.primitiveValue().toCharArray()) {
if (ch < 32 && !(ch == '\r' || ch == '\n' || ch == '\t')) {
// can't get to here with xml - the parser fails if you try
badChars.add(Integer.toHexString(ch));
}
}
warningPlural(errors, "2023-07-26", IssueType.INVALID, e.line(), e.col(), path, badChars.isEmpty(), badChars.size(), I18nConstants.UNICODE_XML_BAD_CHARS, badChars.toString());
}
String regex = context.getExtensionString(ToolingExtensions.EXT_REGEX);
// there's a messy history here - this extension snhould only be on the element definition itself, but for historical reasons