Add support for NDJSON format, and fix bug where JSON parser was not checking for the end of content

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/JsonParser.java

@@ -134,6 +134,10 @@ public class JsonParser extends ParserBase {
 
   @Override
   public List<ValidatedFragment> parse(InputStream inStream) throws IOException, FHIRException {
+    return parse(inStream, 0);
+  }
+  
+  public List<ValidatedFragment> parse(InputStream inStream, int line) throws IOException, FHIRException {
 //    long start = System.currentTimeMillis();
     byte[] content = TextFile.streamToBytes(inStream);
     ValidatedFragment focusFragment = new ValidatedFragment(ValidatedFragment.FOCUS_NAME, "json", content, false);
@@ -146,12 +150,12 @@ public class JsonParser extends ParserBase {
     
     if (policy == ValidationPolicy.EVERYTHING) {
       try {
-        obj = org.hl7.fhir.utilities.json.parser.JsonParser.parseObject(source, true, true); 
+        obj = org.hl7.fhir.utilities.json.parser.JsonParser.parseObject(source, true, true, line); 
       } catch (Exception e) {
-        logError(focusFragment.getErrors(), ValidationMessage.NO_RULE_DATE, -1, -1,context.formatMessage(I18nConstants.DOCUMENT), IssueType.INVALID, context.formatMessage(I18nConstants.ERROR_PARSING_JSON_, e.getMessage()), IssueSeverity.FATAL);
+        logError(focusFragment.getErrors(), ValidationMessage.NO_RULE_DATE, -1, -1, null, IssueType.INVALID, context.formatMessage(I18nConstants.ERROR_PARSING_JSON_, e.getMessage()), IssueSeverity.FATAL);
       }
     } else {
-      obj = org.hl7.fhir.utilities.json.parser.JsonParser.parseObject(source, true, true); 
+      obj = org.hl7.fhir.utilities.json.parser.JsonParser.parseObject(source, true, true, line); 
     }
     
     if (obj != null) {

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Manager.java

@@ -47,7 +47,7 @@ import org.hl7.fhir.r5.model.StructureDefinition;
 public class Manager {
 
   //TODO use EnumMap
-  public enum FhirFormat { XML, JSON, TURTLE, TEXT, VBAR, SHC, SHL, FML; 
+  public enum FhirFormat { XML, JSON, TURTLE, TEXT, VBAR, SHC, SHL, FML, NDJSON; 
     // SHC = smart health cards, including as text versions of QR codes
     // SHL = smart health links, also a text version of the QR code
     
@@ -69,6 +69,8 @@ public class Manager {
           return "shl";
         case FML:
           return "fml";
+        case NDJSON:
+          return "ndjson";
       }
       return null;
     }
@@ -91,6 +93,8 @@ public class Manager {
           return SHL;
         case "fml":
           return FML;
+        case "ndjson":
+          return NDJSON;
       }
       return null;
     }
@@ -127,6 +131,7 @@ public class Manager {
     }
     switch (format) {
     case JSON : return new JsonParser(context);
+    case NDJSON : return new NDJsonParser(context);
     case XML : return new XmlParser(context);
     case TURTLE : return new TurtleParser(context);
     case VBAR : return new VerticalBarParser(context);

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/NDJsonParser.java

@@ -0,0 +1,120 @@
+package org.hl7.fhir.r5.elementmodel;
+
+import java.io.ByteArrayInputStream;
+
+/*
+  Copyright (c) 2011+, HL7, Inc.
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without modification, 
+  are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this 
+     list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, 
+     this list of conditions and the following disclaimer in the documentation 
+     and/or other materials provided with the distribution.
+ * Neither the name of HL7 nor the names of its contributors may be used to 
+     endorse or promote products derived from this software without specific 
+     prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
+  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
+  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
+  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
+  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+  POSSIBILITY OF SUCH DAMAGE.
+
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r5.conformance.profile.ProfileUtilities;
+import org.hl7.fhir.r5.context.IWorkerContext;
+import org.hl7.fhir.r5.formats.IParser.OutputStyle;
+import org.hl7.fhir.r5.formats.JsonCreator;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.i18n.I18nConstants;
+import org.hl7.fhir.utilities.validation.ValidationMessage;
+import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
+import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
+
+
+public class NDJsonParser extends ParserBase {
+
+  public NDJsonParser(IWorkerContext context, ProfileUtilities utilities) {
+    super(context, utilities);
+  }
+
+  public NDJsonParser(IWorkerContext context) {
+    super(context);
+  }
+  
+  private ValidatedFragment processLine(int lineCount, String line) throws FHIRException, IOException {
+    List<ValidatedFragment> list = new JsonParser(context).parse(new ByteArrayInputStream(line.getBytes(StandardCharsets.UTF_8)), lineCount);
+    return list.get(0);
+  }
+
+  @Override
+  public List<ValidatedFragment> parse(InputStream inStream) throws IOException, FHIRException {
+    String source = TextFile.streamToString(inStream);
+    int length = source.length();
+    int start = 0;
+    int cursor = start;
+    int lineCount = 0;
+
+    List<ValidatedFragment> res = new ArrayList<>();
+    while (cursor < length) {
+      while (cursor < length && source.charAt(cursor) != '\n') {
+        cursor++;
+      }
+      if (cursor < length) {
+        String line = source.substring(start, cursor);
+        if (line.endsWith("\r")) {
+          line = line.substring(0, line.length()-1);
+        }
+        if (Utilities.noString(line.trim())) {
+          ValidatedFragment vf = new ValidatedFragment(ValidatedFragment.ITEM_NAME, null, null, false);
+          logError(vf.getErrors(), "2024-06-30", lineCount+1, 1, null, IssueType.INFORMATIONAL, context.formatMessage(I18nConstants.NDJSON_EMPTY_LINE_WARNING), IssueSeverity.WARNING);
+          res.add(vf);
+        } else {
+          res.add(processLine(lineCount, line));
+        }
+        start = cursor+1;
+      } else if (cursor > start) {
+        String line = source.substring(start, cursor);
+        if (line.endsWith("\r")) {
+          line = line.substring(0, line.length()-1);
+        }
+        if (Utilities.noString(line.trim())) {
+          ValidatedFragment vf = new ValidatedFragment(ValidatedFragment.ITEM_NAME, null, null, false);
+          logError(vf.getErrors(), "2024-06-30", lineCount+1, 1, null, IssueType.INFORMATIONAL, context.formatMessage(I18nConstants.NDJSON_EMPTY_LINE_WARNING), IssueSeverity.WARNING);
+          res.add(vf);
+        } else {
+          res.add(processLine(lineCount, line));
+        }
+        start = cursor+1;
+      }
+      lineCount++;
+      cursor++;
+    }
+    return res;
+  }
+  
+  @Override
+  public void compose(Element e, OutputStream stream, OutputStyle style, String identity) throws FHIRException, IOException {
+    throw new Error("Not done yet");
+  }
+
+}

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/ValidatedFragment.java

@@ -27,6 +27,7 @@ public class ValidatedFragment {
   @Getter
   private final boolean isDerivedContent;
   public final static String FOCUS_NAME = "focus";
+  public final static String ITEM_NAME = "item";
 
   @Getter
   private List<ValidationMessage> errors = new ArrayList<>();

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/I18nConstants.java

@@ -1091,4 +1091,5 @@ public class I18nConstants {
   public static final String IG_DEPENDENCY_VERSION_WARNING = "IG_DEPENDENCY_VERSION_WARNING";
   public static final String IG_DEPENDENCY_EXCEPTION = "IG_DEPENDENCY_EXCEPTION";
   public static final String IG_DEPENDENCY_PACKAGE_UNKNOWN = "IG_DEPENDENCY_PACKAGE_UNKNOWN";
+  public static final String NDJSON_EMPTY_LINE_WARNING = "NDJSON_EMPTY_LINE_WARNING";
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonLexer.java

@@ -51,12 +51,12 @@ public class JsonLexer {
   private boolean isUnquoted;
   private String sourceName;
 
-  public JsonLexer(String source, boolean allowComments, boolean allowUnquotedStrings) throws IOException {
+  public JsonLexer(String source, boolean allowComments, boolean allowUnquotedStrings, int line) throws IOException {
     this.source = source;
     this.allowComments = allowComments;
     this.allowUnquotedStrings = allowUnquotedStrings;
     cursor = -1;
-    location = new JsonLocationData(1, 1);  
+    location = new JsonLocationData(line+1, 1);  
     start();
   }
 

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonParser.java

@@ -52,6 +52,15 @@ import org.hl7.fhir.utilities.json.parser.JsonLexer.TokenType;
  */
 public class JsonParser {
 
+  protected JsonParser() {
+    super();
+  }
+
+  protected JsonParser(int line) {
+    super();
+    this.line = line;
+  }
+
   public static JsonObject parseObject(InputStream stream) throws IOException, JsonException {
     return new JsonParser().parseJsonObject(TextFile.streamToString(stream), false, false);
   }
@@ -107,6 +116,10 @@ public class JsonParser {
     return new JsonParser().parseJsonObject(source, isJson5, allowDuplicates);
   }
   
+  public static JsonObject parseObject(String source, boolean isJson5, boolean allowDuplicates, int line) throws IOException, JsonException {
+    return new JsonParser(line).parseJsonObject(source, isJson5, allowDuplicates);
+  }
+  
   // ================================================================
   
   public static JsonElement parse(InputStream stream) throws IOException, JsonException {
@@ -221,6 +234,7 @@ public class JsonParser {
   private boolean itemUnquoted;
   private boolean valueUnquoted;
   private String sourceName;
+  private int line = 0;
 
   private JsonObject parseJsonObject(String source, boolean isJson5, boolean allowDuplicates) throws IOException, JsonException {
     this.allowDuplicates = allowDuplicates;
@@ -231,7 +245,7 @@ public class JsonParser {
   }
 
   private JsonObject parseSource(String source) throws IOException, JsonException {
-    lexer = new JsonLexer(source, allowComments, allowUnquotedStrings);
+    lexer = new JsonLexer(source, allowComments, allowUnquotedStrings, line);
     lexer.setSourceName(sourceName);
     JsonObject result = new JsonObject();
     lexer.takeComments(result);
@@ -249,8 +263,14 @@ public class JsonParser {
     if (lexer.getType() != TokenType.Close) {
       parseProperty();
       readObject("$", result, true);
+      result.setEnd(endProperty != null ? endProperty.copy() : lexer.getLocation().copy());
+    } else {
+      result.setEnd(endProperty != null ? endProperty.copy() : lexer.getLocation().copy());
+      lexer.next();
+    }
+    if (lexer.getType() != TokenType.Eof) {
+      throw lexer.error("Unexpected content at end of JSON: "+lexer.getType().toString());
     }
-    result.setEnd(endProperty != null ? endProperty.copy() : lexer.getLocation().copy());
     return result;
   }
   
@@ -263,7 +283,7 @@ public class JsonParser {
   }
   
   private JsonElement parseSourceElement(String source) throws IOException, JsonException {
-    lexer = new JsonLexer(source, allowComments, allowUnquotedStrings);
+    lexer = new JsonLexer(source, allowComments, allowUnquotedStrings, line);
     switch (lexer.getType()) {
     case Boolean:
       JsonBoolean bool = new JsonBoolean(lexer.getValue().equals("true"));

org.hl7.fhir.utilities/src/main/resources/Messages.properties

@@ -1119,3 +1119,4 @@ IG_DEPENDENCY_PACKAGE_UNKNOWN = The package {0} could not be found so the versio
 IG_DEPENDENCY_VERSION_ERROR = The ImplementationGuide is based on FHIR version {0} but package {1} is based on FHIR version {2}. Use the package {3} instead
 IG_DEPENDENCY_VERSION_WARNING = The ImplementationGuide is based on FHIR version {0} but package {1} is based on FHIR version {2}. In general, this version mismatch should be avoided - some tools will try to make this work with variable degrees of success, but others will not even try
 IG_DEPENDENCY_EXCEPTION = Exception checking package version consistency: {0}
+NDJSON_EMPTY_LINE_WARNING = The NDJSON source contains an empty line. This may not be accepted by some processors