Fix comment parsing and add focus to FHIRPath function extensions

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/FHIRLexer.java

@@ -146,23 +146,8 @@ public class FHIRLexer {
   }
 
   public void next() throws FHIRLexerException {
+    skipWhitespaceAndComments();
     current = null;
-    boolean last13 = false;
-    while (cursor < source.length() && Character.isWhitespace(source.charAt(cursor))) {
-      if (source.charAt(cursor) == '\r') {
-        currentLocation.setLine(currentLocation.getLine() + 1);
-        currentLocation.setColumn(1);
-        last13 = true;
-      } else if (!last13 && (source.charAt(cursor) == '\n')) {
-        currentLocation.setLine(currentLocation.getLine() + 1);
-        currentLocation.setColumn(1);
-        last13 = false;
-      } else {
-        last13 = false;
-        currentLocation.setColumn(currentLocation.getColumn() + 1);
-      }
-      cursor++;
-    }
     currentStart = cursor;
     currentStartLocation = currentLocation;
     if (cursor < source.length()) {
@@ -208,9 +193,8 @@ public class FHIRLexer {
       } else if (ch == '/') {
         cursor++;
         if (cursor < source.length() && (source.charAt(cursor) == '/')) {
-          cursor++;
+          // this is en error - should already have been skipped
-          while (cursor < source.length() && !((source.charAt(cursor) == '\r') || source.charAt(cursor) == '\n')) 
+          error("This shoudn't happen?");
-            cursor++;
         }
         current = source.substring(currentStart, cursor);
       } else if (ch == '$') {
@@ -297,6 +281,33 @@ public class FHIRLexer {
   }
 
 
+  private void skipWhitespaceAndComments() {
+    boolean last13 = false;
+    boolean done = false;
+    while (cursor < source.length() && !done) {
+      if (cursor < source.length() -1 && "//".equals(source.substring(cursor, cursor+2))) {
+        while (cursor < source.length() && !((source.charAt(cursor) == '\r') || source.charAt(cursor) == '\n')) 
+          cursor++;        
+      } else if (cursor < source.length() - 1 && "/*".equals(source.substring(cursor, cursor+2))) {
+        while (cursor < source.length() - 1 && !"*/".equals(source.substring(cursor, cursor+2))) { 
+          last13 = currentLocation.checkChar(source.charAt(cursor), last13);
+          cursor++;        
+        }
+        if (cursor >= source.length() -1) {
+          error("Unfinished comment");
+        } else {
+          cursor = cursor + 2;
+        }
+      } else if (Character.isWhitespace(source.charAt(cursor))) {
+        last13 = currentLocation.checkChar(source.charAt(cursor), last13);
+        cursor++;
+      } else {
+        done = true;
+      }
+    }
+  }
+
+  
   private boolean isDateChar(char ch,int start) {
     int eot = source.charAt(start+1) == 'T' ? 10 : 20;
     

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/FHIRPathEngine.java

@@ -233,7 +233,7 @@ public class FHIRPathEngine {
      * @param parameters
      * @return
      */
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters);
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters);
     
     /**
      * Implementation of resolve() function. Passed a string, return matching resource, if one is known - else null
@@ -2858,7 +2858,7 @@ public class FHIRPathEngine {
       List<List<Base>> params = new ArrayList<List<Base>>();
       for (ExpressionNode p : exp.getParameters()) 
         params.add(execute(context, focus, p, true));
-      return hostServices.executeFunction(context.appInfo, exp.getName(), params);
+      return hostServices.executeFunction(context.appInfo, focus, exp.getName(), params);
     }
     default:
       throw new Error("not Implemented yet");

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/LiquidEngine.java

@@ -405,11 +405,11 @@ public class LiquidEngine implements IEvaluationContext {
   }
 
   @Override
-  public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+  public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
     if (externalHostServices == null)
       return null;
     LiquidEngineContext ctxt = (LiquidEngineContext) appContext;
-    return externalHostServices.executeFunction(ctxt.externalContext, functionName, parameters);
+    return externalHostServices.executeFunction(ctxt.externalContext, focus, functionName, parameters);
   }
 
   @Override

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/utils/StructureMapUtilities.java

@@ -202,7 +202,7 @@ public class StructureMapUtilities {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       throw new Error("Not Implemented Yet");
     }
 

org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/FHIRPathTests.java

@@ -64,7 +64,7 @@ public class FHIRPathTests {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       throw new NotImplementedException("Not done yet (FHIRPathTestEvaluationServices.executeFunction), when item is element");
     }
 

org.hl7.fhir.r4/src/test/java/org/hl7/fhir/r4/test/SnapShotGenerationTests.java

@@ -316,7 +316,7 @@ public class SnapShotGenerationTests {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       if ("fixture".equals(functionName)) {
         String id = fp.convertToString(parameters.get(0));
         Resource res = fetchFixture(id);

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/comparison/ComparisonRenderer.java

@@ -242,7 +242,7 @@ public class ComparisonRenderer implements IEvaluationContext {
   }
 
   @Override
-  public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+  public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
     return null;
   }
 

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/utils/FHIRLexer.java

@@ -1,5 +1,10 @@
 package org.hl7.fhir.r5.utils;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import org.apache.poi.xssf.model.Comments;
 import org.hl7.fhir.exceptions.FHIRException;
 
 /*
@@ -34,6 +39,7 @@ import org.hl7.fhir.exceptions.FHIRException;
 
 
 import org.hl7.fhir.r5.model.ExpressionNode;
+import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
 import org.hl7.fhir.utilities.SourceLocation;
 import org.hl7.fhir.utilities.Utilities;
 
@@ -65,6 +71,7 @@ public class FHIRLexer {
   private int cursor;
   private int currentStart;
   private String current;
+  private List<String> comments = new ArrayList<>();
   private SourceLocation currentLocation;
   private SourceLocation currentStartLocation;
   private int id;
@@ -146,23 +153,8 @@ public class FHIRLexer {
   }
 
   public void next() throws FHIRLexerException {
+    skipWhitespaceAndComments();
     current = null;
-    boolean last13 = false;
-    while (cursor < source.length() && Character.isWhitespace(source.charAt(cursor))) {
-      if (source.charAt(cursor) == '\r') {
-        currentLocation.setLine(currentLocation.getLine() + 1);
-        currentLocation.setColumn(1);
-        last13 = true;
-      } else if (!last13 && (source.charAt(cursor) == '\n')) {
-        currentLocation.setLine(currentLocation.getLine() + 1);
-        currentLocation.setColumn(1);
-        last13 = false;
-      } else {
-        last13 = false;
-        currentLocation.setColumn(currentLocation.getColumn() + 1);
-      }
-      cursor++;
-    }
     currentStart = cursor;
     currentStartLocation = currentLocation;
     if (cursor < source.length()) {
@@ -208,9 +200,8 @@ public class FHIRLexer {
       } else if (ch == '/') {
         cursor++;
         if (cursor < source.length() && (source.charAt(cursor) == '/')) {
-          cursor++;
+          // this is en error - should already have been skipped
-          while (cursor < source.length() && !((source.charAt(cursor) == '\r') || source.charAt(cursor) == '\n')) 
+          error("This shoudn't happen?");
-            cursor++;
         }
         current = source.substring(currentStart, cursor);
       } else if (ch == '$') {
@@ -296,6 +287,37 @@ public class FHIRLexer {
     }
   }
 
+  private void skipWhitespaceAndComments() {
+    comments.clear();
+    boolean last13 = false;
+    boolean done = false;
+    while (cursor < source.length() && !done) {
+      if (cursor < source.length() -1 && "//".equals(source.substring(cursor, cursor+2))) {
+        int start = cursor+2;
+        while (cursor < source.length() && !((source.charAt(cursor) == '\r') || source.charAt(cursor) == '\n')) { 
+          cursor++;        
+        }
+        comments.add(source.substring(start, cursor).trim());
+      } else if (cursor < source.length() - 1 && "/*".equals(source.substring(cursor, cursor+2))) {
+        int start = cursor+2;
+        while (cursor < source.length() - 1 && !"*/".equals(source.substring(cursor, cursor+2))) { 
+          last13 = currentLocation.checkChar(source.charAt(cursor), last13);
+          cursor++;        
+        }
+        if (cursor >= source.length() -1) {
+          error("Unfinished comment");
+        } else {
+          comments.add(source.substring(start, cursor).trim());
+          cursor = cursor + 2;
+        }
+      } else if (Character.isWhitespace(source.charAt(cursor))) {
+        last13 = currentLocation.checkChar(source.charAt(cursor), last13);
+        cursor++;
+      } else {
+        done = true;
+      }
+    }
+  }
   
   private boolean isDateChar(char ch,int start) {
     int eot = source.charAt(start+1) == 'T' ? 10 : 20;
@@ -321,9 +343,31 @@ public class FHIRLexer {
     this.current = current;
   }
 
-  public boolean hasComment() {
+  public boolean hasComments() {
-    return !done() && current.startsWith("//");
+    return comments.size() > 0;
   }
+
+  public List<String> getComments() {
+    return comments;
+  }
+
+  public String getAllComments() {
+    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder("\r\n");
+    b.addAll(comments);
+    comments.clear();
+    return b.toString();
+  }
+
+  public String getFirstComment() {
+    if (hasComments()) {
+      String s = comments.get(0);
+      comments.remove(0);
+      return s;      
+    } else {
+      return null;
+    }
+  }
+
   public boolean hasToken(String kw) {
     return !done() && kw.equals(current);
   }
@@ -472,10 +516,6 @@ public class FHIRLexer {
     return b.toString();
   }
   
-  void skipComments() throws FHIRLexerException {
-    while (!done() && hasComment())
-      next();
-  }
   public int getCurrentStart() {
     return currentStart;
   }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/utils/FHIRPathEngine.java

@@ -285,7 +285,7 @@ public class FHIRPathEngine {
      * @param parameters
      * @return
      */
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters);
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters);
     
     /**
      * Implementation of resolve() function. Passed a string, return matching resource, if one is known - else null
@@ -3463,7 +3463,7 @@ public class FHIRPathEngine {
       for (ExpressionNode p : exp.getParameters()) {
         params.add(execute(context, focus, p, true));
       }
-      return hostServices.executeFunction(context.appInfo, exp.getName(), params);
+      return hostServices.executeFunction(context.appInfo, focus, exp.getName(), params);
     }
     default:
       throw new Error("not Implemented yet");

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/utils/LiquidEngine.java

@@ -632,11 +632,11 @@ public class LiquidEngine implements IEvaluationContext {
   }
 
   @Override
-  public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+  public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
     if (externalHostServices == null)
       return null;
     LiquidEngineContext ctxt = (LiquidEngineContext) appContext;
-    return externalHostServices.executeFunction(ctxt.externalContext, functionName, parameters);
+    return externalHostServices.executeFunction(ctxt.externalContext, focus, functionName, parameters);
   }
 
   @Override

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/utils/StructureMapUtilities.java

@@ -202,7 +202,7 @@ public class StructureMapUtilities {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       throw new Error("Not Implemented Yet");
     }
 
@@ -748,15 +748,12 @@ public class StructureMapUtilities {
 		FHIRLexer lexer = new FHIRLexer(text, srcName);
 		if (lexer.done())
 			throw lexer.error("Map Input cannot be empty");
-		lexer.skipComments();
 		lexer.token("map");
 		StructureMap result = new StructureMap();
 		result.setUrl(lexer.readConstant("url"));
 		lexer.token("=");
 		result.setName(lexer.readConstant("name"));
-		if (lexer.hasComment()) {
+    result.setDescription(lexer.getAllComments());
-	    result.setDescription(getMultiLineComments(lexer));
-		}
 		while (lexer.hasToken("conceptmap"))
 			parseConceptMap(result, lexer);
 
@@ -782,7 +779,6 @@ public class StructureMapUtilities {
 		map.setStatus(PublicationStatus.DRAFT); // todo: how to add this to the text format
 		result.getContained().add(map);
 		lexer.token("{");
-		lexer.skipComments();
 		//	  lexer.token("source");
 		//	  map.setSource(new UriType(lexer.readConstant("source")));
 		//	  lexer.token("target");
@@ -824,25 +820,11 @@ public class StructureMapUtilities {
 			tgt.setCode(lexer.take());
 			if (tgt.getCode().startsWith("\""))
 			  tgt.setCode(lexer.processConstant(tgt.getCode()));
-      if (lexer.hasComment())
+       tgt.setComment(lexer.getFirstComment());
-        tgt.setComment(lexer.take().substring(2).trim());
 		}
 		lexer.token("}");
 	}
 	
-	private String getMultiLineComments(FHIRLexer lexer) {
-	  String comment = null;
-    while (lexer.hasComment()) { 
-      String newComment = lexer.take().substring(2).trim();
-      if (comment == null) {
-        comment = newComment;
-      } else {
-        comment += "\r\n"+newComment;
-      }
-    }
-	  return comment;
-	}
-
 	private ConceptMapGroupComponent getGroup(ConceptMap map, String srcs, String tgts) {
 	  for (ConceptMapGroupComponent grp : map.getGroup()) {
 	    if (grp.getSource().equals(srcs)) 
@@ -895,9 +877,7 @@ public class StructureMapUtilities {
 		lexer.token("as");
 		st.setMode(StructureMapModelMode.fromCode(lexer.take()));
 		lexer.skipToken(";");
-    if (lexer.hasComment() && currentLine == lexer.getCurrentLocation().getLine()) {
+   	st.setDocumentation(lexer.getFirstComment());
-			st.setDocumentation(lexer.take().substring(2).trim());
-		}
 	}
 
 	private void parseImports(StructureMap result, FHIRLexer lexer) throws FHIRException {
@@ -905,19 +885,11 @@ public class StructureMapUtilities {
 		int currentLine = lexer.getCurrentLocation().getLine();
 		result.addImport(lexer.readConstant("url"));
 		lexer.skipToken(";");
-		if (lexer.hasComment() && currentLine == lexer.getCurrentLocation().getLine()) {
+  	lexer.getFirstComment();
-			lexer.next();
-		}
 	}
 
 	private void parseGroup(StructureMap result, FHIRLexer lexer) throws FHIRException {
-	  String comment = null;
+	  String comment = lexer.getAllComments();
-    if (lexer.hasComment()) {
-      comment = getMultiLineComments(lexer);
-      if (lexer.done()) {
-        return ;
-      }
-    }
 		lexer.token("group");
 		StructureMapGroupComponent group = result.addGroup();
 		if (comment != null) {
@@ -975,7 +947,6 @@ public class StructureMapUtilities {
         parseRule(result, group.getRule(), lexer, true);
       }
 		} else {
-	    lexer.skipComments();
 		  while (lexer.hasToken("input")) 
 		    parseInput(group, lexer, false);
 		  while (!lexer.hasToken("endgroup")) {
@@ -1003,11 +974,8 @@ public class StructureMapUtilities {
 		if (!newFmt) {
   		lexer.token("as");
 	  	input.setMode(StructureMapInputMode.fromCode(lexer.take()));
-		  if (lexer.hasComment()) {
+  	  input.setDocumentation(lexer.getFirstComment());
-			  input.setDocumentation(lexer.take().substring(2).trim());
-	  	}
 	  	lexer.skipToken(";");
-		  lexer.skipComments();
 		}
 	}
 
@@ -1018,12 +986,7 @@ public class StructureMapUtilities {
 		  lexer.token(":");
 		  lexer.token("for");
     } else {
-      if (lexer.hasComment()) {
+      rule.setDocumentation(lexer.getFirstComment());
-        rule.setDocumentation(this.getMultiLineComments(lexer));
-        if (lexer.hasToken("}")) {
-          return ; // catched a comment at the end
-        }
-      }
     }
     list.add(rule);
 		boolean done = false;
@@ -1062,8 +1025,9 @@ public class StructureMapUtilities {
 						lexer.next();
 				}
 			}
-		} else if (lexer.hasComment()) {
+		} 
-			rule.setDocumentation(getMultiLineComments(lexer));
+		if (!rule.hasDocumentation() && lexer.hasComments()) {
+		  rule.setDocumentation(lexer.getFirstComment());
 		}
 		if (isSimpleSyntax(rule)) {
 		  rule.getSourceFirstRep().setVariable(AUTO_VAR_NAME);

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/FHIRPathTests.java

@@ -72,7 +72,7 @@ public class FHIRPathTests {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       throw new NotImplementedException("Not done yet (FHIRPathTestEvaluationServices.executeFunction), when item is element");
     }
 

org.hl7.fhir.r5/src/test/java/org/hl7/fhir/r5/test/SnapShotGenerationTests.java

@@ -380,7 +380,7 @@ public class SnapShotGenerationTests {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       if ("fixture".equals(functionName)) {
         String id = fp.convertToString(parameters.get(0));
         Resource res = fetchFixture(id);

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/SourceLocation.java

@@ -24,4 +24,23 @@ public class SourceLocation {
   public String toString() {
     return Integer.toString(line)+", "+Integer.toString(column);
   }
+  
+  public void newLine() {
+    setLine(getLine() + 1);
+    setColumn(1);
+  }
+  public boolean checkChar(char ch, boolean last13) {
+    if (ch == '\r') {
+      newLine();
+      return true;
+    } else if (ch == '\n') {
+      if (!last13) {
+        newLine();
+      }
+      return false;
+    } else {
+      setColumn(getColumn() + 1);
+      return false;
+    }
+  }
 }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/instance/InstanceValidator.java

@@ -226,7 +226,7 @@ public class InstanceValidator extends BaseValidator implements IResourceValidat
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       throw new Error(context.formatMessage(I18nConstants.NOT_DONE_YET_VALIDATORHOSTSERVICESEXECUTEFUNCTION));
     }
 

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/SnapShotGenerationXTests.java

@@ -334,7 +334,7 @@ public class SnapShotGenerationXTests {
     }
 
     @Override
-    public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+    public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
       if ("fixture".equals(functionName)) {
         String id = fp.convertToString(parameters.get(0));
         Resource res = fetchFixture(id);

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ValidationTests.java

@@ -435,7 +435,7 @@ public class ValidationTests implements IEvaluationContext, IValidatorResourceFe
   }
 
   @Override
-  public List<Base> executeFunction(Object appContext, String functionName, List<List<Base>> parameters) {
+  public List<Base> executeFunction(Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
     return null;
   }