fix issues in testing

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/context/SimpleWorkerContext.java

@@ -78,6 +78,9 @@ import org.hl7.fhir.r5.utils.IResourceValidator;
 import org.hl7.fhir.utilities.CSFileInputStream;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.VersionUtilities;
+import org.hl7.fhir.utilities.cache.BasePackageCacheManager;
+import org.hl7.fhir.utilities.cache.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.cache.NpmPackage;
 import org.hl7.fhir.utilities.cache.NpmPackage.PackageResourceInformation;
 import org.hl7.fhir.utilities.i18n.I18nConstants;
@@ -141,7 +144,7 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
   private boolean ignoreProfileErrors;
   private boolean progress;
   private List<String> loadedPackages = new ArrayList<String>();
-  
+
   public SimpleWorkerContext() throws FileNotFoundException, IOException, FHIRException {
     super();
   }
@@ -170,6 +173,11 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
     validatorFactory = other.validatorFactory;
   }
 
+
+  public List<String> getLoadedPackages() {
+    return loadedPackages;
+  }
+
   // -- Initializations
 	/**
 	 * Load the working context from the validation pack
@@ -189,13 +197,9 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
   }
 
   public static SimpleWorkerContext fromPackage(NpmPackage pi, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException {
-    return fromPackage(pi, allowDuplicates, null);
-  }
-  
-  public static SimpleWorkerContext fromPackage(NpmPackage pi, boolean allowDuplicates, ILoadFilter filter) throws FileNotFoundException, IOException, FHIRException {
     SimpleWorkerContext res = new SimpleWorkerContext();
     res.setAllowLoadingDuplicates(allowDuplicates);
-    res.loadFromPackage(pi, null, filter);
+    res.loadFromPackage(pi, null);
     return res;
   }
 
@@ -206,14 +210,10 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
   }
 
   public static SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException {
-    return fromPackage(pi, loader, null);  
-  }
-  
-  public static SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader, ILoadFilter filter) throws FileNotFoundException, IOException, FHIRException {
     SimpleWorkerContext res = new SimpleWorkerContext();
     res.setAllowLoadingDuplicates(true);
     res.version = pi.getNpm().get("version").getAsString();
-    res.loadFromPackage(pi, loader, filter);
+    res.loadFromPackage(pi, loader);
     res.finishLoading();
     return res;
   }
@@ -356,46 +356,62 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
 		loadFromStream(new CSFileInputStream(path), loader);
 	}
   
-  public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, ILoadFilter filter) throws IOException  {
+//  public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, ILoadFilter filter) throws IOException  {
+//    if (progress) {
+//      System.out.println("Load Package "+pi.name()+"#"+pi.version());
+//    }
+//    loadedPackages.add(pi.id()+"#"+pi.version());
+//    String [] types = loader != null ? loader.getTypes() : new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem" }; 
+//    for (PackageResourceInformation pri : pi.listIndexedResources(types)) {
+//       try {
+//         registerResourceFromPackage(new PackageResourceLoader(pri, loader), new PackageVersion(pi.id(), pi.version()));
+//      } catch (FHIRException e) {
+//        throw new FHIRException(formatMessage(I18nConstants.ERROR_READING__FROM_PACKAGE__, pri.getFilename(), pi.name(), pi.version(), e.getMessage()), e);
+//      }
+//    }
+//    for (String s : pi.list("other")) {
+//      binaries.put(s, TextFile.streamToBytes(pi.load("other", s)));
+//    }
+//    if (version == null) {
+//      version = pi.version();
+//    }
+//  }
+
+  @Override
+  public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, String... types) throws FileNotFoundException, IOException, FHIRException {
     if (progress) {
       System.out.println("Load Package "+pi.name()+"#"+pi.version());
     }
     loadedPackages.add(pi.id()+"#"+pi.version());
-    String [] types = loader != null ? loader.getTypes() : new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem" }; 
-    for (PackageResourceInformation pri : pi.listIndexedResources(types)) {
-       try {
-         registerResourceFromPackage(new PackageResourceLoader(pri, loader), new PackageVersion(pi.id(), pi.version()));
-      } catch (FHIRException e) {
-        throw new FHIRException(formatMessage(I18nConstants.ERROR_READING__FROM_PACKAGE__, pri.getFilename(), pi.name(), pi.version(), e.getMessage()), e);
-      }
-    }
-    for (String s : pi.list("other")) {
-      binaries.put(s, TextFile.streamToBytes(pi.load("other", s)));
-    }
-    if (version == null) {
-      version = pi.version();
-    }
-  }
 
-  @Override
+    
-	public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, String... types) throws FileNotFoundException, IOException, FHIRException {
-	  if (progress) {
-	    System.out.println("Load Package "+pi.name()+"#"+pi.version());
-	  }
-    loadedPackages.add(pi.id()+"#"+pi.version());
     if (types.length == 0 &&  loader != null) {
       types = loader.getTypes();
     }
-	  if (types.length == 0) {
+    if (VersionUtilities.isR2Ver(pi.fhirVersion())) {
-	    types = new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem" };
+      // can't lazy load R2 because of valueset/codesystem implementation
-	  }
+      if (types.length == 0) {
-    for (PackageResourceInformation pri : pi.listIndexedResources(types)) {
+        types = new String[] { "StructureDefinition", "ValueSet", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem" };
-      try {
-        registerResourceFromPackage(new PackageResourceLoader(pri, loader), new PackageVersion(pi.id(), pi.version()));
-      } catch (FHIRException e) {
-        throw new FHIRException(formatMessage(I18nConstants.ERROR_READING__FROM_PACKAGE__, pri.getFilename(), pi.name(), pi.version(), e.getMessage()), e);
       }
-	  }
+      for (String s : pi.listResources(loader.getTypes())) {
+        try {
+          loadDefinitionItem(s, pi.load("package", s), loader, null, new PackageVersion(pi.id(), pi.version()));
+        } catch (FHIRException e) {
+          throw new FHIRException(formatMessage(I18nConstants.ERROR_READING__FROM_PACKAGE__, s, pi.name(), pi.version(), e.getMessage()), e);
+        }
+      }
+    } else {
+      if (types.length == 0) {
+        types = new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire", "ConceptMap", "StructureMap", "NamingSystem" };
+      }
+      for (PackageResourceInformation pri : pi.listIndexedResources(types)) {
+        try {
+          registerResourceFromPackage(new PackageResourceLoader(pri, loader), new PackageVersion(pi.id(), pi.version()));
+        } catch (FHIRException e) {
+          throw new FHIRException(formatMessage(I18nConstants.ERROR_READING__FROM_PACKAGE__, pri.getFilename(), pi.name(), pi.version(), e.getMessage()), e);
+        }
+      }
+    }
 	  for (String s : pi.list("other")) {
 	    binaries.put(s, TextFile.streamToBytes(pi.load("other", s)));
 	  }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/terminologies/ValueSetExpanderSimple.java

@@ -83,6 +83,9 @@ import org.hl7.fhir.r5.model.CodeSystem;
 import org.hl7.fhir.r5.model.CodeSystem.CodeSystemContentMode;
 import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
 import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
+import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
+import org.hl7.fhir.r5.model.CodeSystem.PropertyComponent;
+import org.hl7.fhir.r5.model.CodeSystem.PropertyType;
 import org.hl7.fhir.r5.model.DataType;
 import org.hl7.fhir.r5.model.DateTimeType;
 import org.hl7.fhir.r5.model.Enumerations.FilterOperator;
@@ -101,11 +104,75 @@ import org.hl7.fhir.r5.model.ValueSet.ValueSetComposeComponent;
 import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionComponent;
 import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
 import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionParameterComponent;
+import org.hl7.fhir.r5.terminologies.ValueSetExpanderSimple.AllConceptsFilter;
+import org.hl7.fhir.r5.terminologies.ValueSetExpanderSimple.IConceptFilter;
+import org.hl7.fhir.r5.terminologies.ValueSetExpanderSimple.PropertyFilter;
 import org.hl7.fhir.r5.utils.ToolingExtensions;
 import org.hl7.fhir.utilities.Utilities;
 
 public class ValueSetExpanderSimple implements ValueSetExpander {
 
+  public class PropertyFilter implements IConceptFilter {
+
+    private ConceptSetFilterComponent filter;
+    private PropertyComponent property;
+
+    public PropertyFilter(ConceptSetFilterComponent fc, PropertyComponent propertyDefinition) {
+      this.filter = fc;
+      this.property = propertyDefinition;
+    }
+
+    @Override
+    public boolean includeConcept(CodeSystem cs, ConceptDefinitionComponent def) {
+      ConceptPropertyComponent pc = getPropertyForConcept(def);
+      if (pc != null) {
+        String v = pc.getValue().isPrimitive() ? pc.getValue().primitiveValue() : null;
+        switch (filter.getOp()) {
+        case DESCENDENTOF: throw new FHIRException("not supported yet");
+        case EQUAL: return filter.getValue().equals(v);
+        case EXISTS: throw new FHIRException("not supported yet");
+        case GENERALIZES: throw new FHIRException("not supported yet");
+        case IN: throw new FHIRException("not supported yet");
+        case ISA: throw new FHIRException("not supported yet");
+        case ISNOTA: throw new FHIRException("not supported yet");
+        case NOTIN: throw new FHIRException("not supported yet");
+        case NULL: throw new FHIRException("not supported yet");
+        case REGEX: throw new FHIRException("not supported yet");
+        default:
+          throw new FHIRException("Shouldn't get here");        
+        }            
+      } else if (property.getType() == PropertyType.BOOLEAN && filter.getOp() == FilterOperator.EQUAL) {
+        return "false".equals(filter.getValue()); 
+      } else {
+        return false;
+      }
+    }
+
+    private ConceptPropertyComponent getPropertyForConcept(ConceptDefinitionComponent def) {
+      for (ConceptPropertyComponent pc : def.getProperty()) {
+        if (pc.getCode().equals(property.getCode())) {
+          return pc;
+        }
+      }
+      return null;
+    }
+
+  }
+
+  public class AllConceptsFilter implements IConceptFilter {
+
+    @Override
+    public boolean includeConcept(CodeSystem cs, ConceptDefinitionComponent def) {
+      return true;
+    }
+  }
+
+  public interface IConceptFilter {
+
+    boolean includeConcept(CodeSystem cs, ConceptDefinitionComponent def);
+
+  }
+
   private List<ValueSetExpansionContainsComponent> codes = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
   private List<ValueSetExpansionContainsComponent> roots = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
   private Map<String, ValueSetExpansionContainsComponent> map = new HashMap<String, ValueSet.ValueSetExpansionContainsComponent>();
@@ -214,7 +281,7 @@ public class ValueSetExpanderSimple implements ValueSetExpander {
     return list;
   }
 
-  private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, ConceptDefinitionComponent exclusion)  throws FHIRException {
+  private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, ConceptDefinitionComponent exclusion, IConceptFilter filterFunc)  throws FHIRException {
     def.checkNoModifiers("Code in Code System", "expanding");
     if (exclusion != null) {
       if (exclusion.getCode().equals(def.getCode()))
@@ -224,24 +291,25 @@ public class ValueSetExpanderSimple implements ValueSetExpander {
       ValueSetExpansionContainsComponent np = null;
       boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
       boolean inc = CodeSystemUtilities.isInactive(cs, def);
-      if (includeAbstract || !abs)
+      if ((includeAbstract || !abs)  && filterFunc.includeConcept(cs, def)) {
         np = addCode(system, def.getCode(), def.getDisplay(), parent, def.getDesignation(), expParams, abs, inc, filters);
+      }
       for (ConceptDefinitionComponent c : def.getConcept()) {
-        addCodeAndDescendents(cs, system, c, np, expParams, filters, exclusion);
+        addCodeAndDescendents(cs, system, c, np, expParams, filters, exclusion, filterFunc);
       }
       if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
         List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
         for (ConceptDefinitionComponent c : children)
-          addCodeAndDescendents(cs, system, c, np, expParams, filters, exclusion);
+          addCodeAndDescendents(cs, system, c, np, expParams, filters, exclusion, filterFunc);
       }
     } else {
       for (ConceptDefinitionComponent c : def.getConcept()) {
-        addCodeAndDescendents(cs, system, c, null, expParams, filters, exclusion);
+        addCodeAndDescendents(cs, system, c, null, expParams, filters, exclusion, filterFunc);
       }
       if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
         List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
         for (ConceptDefinitionComponent c : children)
-          addCodeAndDescendents(cs, system, c, null, expParams, filters, exclusion);
+          addCodeAndDescendents(cs, system, c, null, expParams, filters, exclusion, filterFunc);
       }
     }
 
@@ -517,7 +585,7 @@ public class ValueSetExpanderSimple implements ValueSetExpander {
     if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
       // special case - add all the code system
       for (ConceptDefinitionComponent def : cs.getConcept()) {
-        addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null);
+        addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter());
       }
       if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
         addFragmentWarning(exp, cs);
@@ -556,14 +624,14 @@ public class ValueSetExpanderSimple implements ValueSetExpander {
         ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
         if (def == null)
           throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
-        addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null);
+        addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter());
       } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISNOTA) {
         // special: all codes in the target code system that are not under the value
         ConceptDefinitionComponent defEx = getConceptForCode(cs.getConcept(), fc.getValue());
         if (defEx == null)
           throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
         for (ConceptDefinitionComponent def : cs.getConcept()) {
-          addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, defEx);
+          addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, defEx, new AllConceptsFilter());
         }
       } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.DESCENDENTOF) {
         // special: all codes in the target code system under the value
@@ -571,11 +639,11 @@ public class ValueSetExpanderSimple implements ValueSetExpander {
         if (def == null)
           throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
         for (ConceptDefinitionComponent c : def.getConcept())
-          addCodeAndDescendents(cs, inc.getSystem(), c, null, expParams, imports, null);
+          addCodeAndDescendents(cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter());
         if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
           List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
           for (ConceptDefinitionComponent c : children)
-            addCodeAndDescendents(cs, inc.getSystem(), c, null, expParams, imports, null);
+            addCodeAndDescendents(cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter());
         }
 
       } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
@@ -590,11 +658,34 @@ public class ValueSetExpanderSimple implements ValueSetExpander {
             }
           }
         }
-      } else
+      } else if (isDefinedProperty(cs, fc.getProperty())) {
+        for (ConceptDefinitionComponent def : cs.getConcept()) {
+          addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null, new PropertyFilter(fc, getPropertyDefinition(cs, fc.getProperty())));
+        }
+      } else {
         throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
+      }
     }
   }
 
+  private PropertyComponent getPropertyDefinition(CodeSystem cs, String property) {
+    for (PropertyComponent cp : cs.getProperty()) {
+      if (cp.getCode().equals(property)) {
+        return cp;
+      }
+    }
+    return null;
+  }
+
+  private boolean isDefinedProperty(CodeSystem cs, String property) {
+    for (PropertyComponent cp : cs.getProperty()) {
+      if (cp.getCode().equals(property)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private void addFragmentWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
     for (Extension ex : cs.getExtensionsByUrl(ToolingExtensions.EXT_EXP_FRAGMENT)) {
       if (ex.getValue().primitiveValue().equals(cs.getUrl())) {

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java

@@ -261,7 +261,6 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   private FilesystemPackageCacheManager pcm;
   private PrintWriter mapLog;
   private boolean debug;
-  private Set<String> loadedIgs = new HashSet<>();
   private IValidatorResourceFetcher fetcher;
   private boolean assumeValidRestReferences;
   private boolean noExtensibleBindingMessages;
@@ -608,10 +607,10 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   }
 
   public Map<String, byte[]> loadPackage(NpmPackage pi) throws Exception {
-    loadedIgs.add(pi.name()+"#"+pi.version());
+    context.getLoadedPackages().add(pi.name()+"#"+pi.version());
     Map<String, byte[]> res = new HashMap<String, byte[]>();
     for (String s : pi.dependencies()) {
-      if (!loadedIgs.contains(s)) {
+      if (! context.getLoadedPackages().contains(s)) {
         if (!VersionUtilities.isCorePackage(s)) {
           System.out.println("+  .. load IG from "+s);
           res.putAll(fetchByPackage(s));
@@ -772,6 +771,13 @@ public class ValidationEngine implements IValidatorResourceFetcher {
   public void loadIg(String src, boolean recursive) throws IOException, FHIRException, Exception {
     NpmPackage npm = pcm.loadPackage(src, null);
     if (npm != null) {
+      for (String s : npm.dependencies()) {
+        if (!context.getLoadedPackages().contains(s)) {
+          if (!VersionUtilities.isCorePackage(s)) {
+            loadIg(s, false);
+          }
+        }
+      }
       context.loadFromPackage(npm, loaderForVersion(npm.fhirVersion()));
     } else {    
       String canonical = null;

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/conversion/tests/R3R4ConversionTests.java

@@ -287,7 +287,7 @@ public class R3R4ConversionTests implements ITransformerServices, IValidatorReso
       return;
 
     pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
-    R3ToR4Loader ldr = new R3ToR4Loader().setPatchUrls(true).setKillPrimitives(true);
+    R3ToR4Loader ldr = (R3ToR4Loader) new R3ToR4Loader().setPatchUrls(true).setKillPrimitives(true);
 
     System.out.println("loading R3");
     contextR3 = new SimpleWorkerContext();