update Mimic importers

org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/context/SimpleWorkerContext.java

@@ -79,6 +79,7 @@ import org.hl7.fhir.exceptions.FHIRFormatError;
 import org.hl7.fhir.utilities.CSFileInputStream;
 import org.hl7.fhir.utilities.OIDUtils;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.cache.NpmPackage;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
 import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
@@ -129,6 +130,27 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
     return res;
   }
 
+  public static SimpleWorkerContext fromPackage(NpmPackage pi, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException {
+    SimpleWorkerContext res = new SimpleWorkerContext();
+    res.setAllowLoadingDuplicates(allowDuplicates);
+    res.loadFromPackage(pi, null);
+    return res;
+  }
+
+  public static SimpleWorkerContext fromPackage(NpmPackage pi) throws FileNotFoundException, IOException, FHIRException {
+    SimpleWorkerContext res = new SimpleWorkerContext();
+    res.loadFromPackage(pi, null);
+    return res;
+  }
+
+  public static SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException {
+    SimpleWorkerContext res = new SimpleWorkerContext();
+    res.setAllowLoadingDuplicates(true);
+    res.version = pi.getNpm().get("version").getAsString();
+    res.loadFromPackage(pi, loader);
+    return res;
+  }
+
   public static SimpleWorkerContext fromPack(String path, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException {
     SimpleWorkerContext res = new SimpleWorkerContext();
     res.allowLoadingDuplicates = allowDuplicates;
@@ -163,6 +185,18 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
 		return res;
 	}
 
+  public static SimpleWorkerContext fromDefinitions(Map<String, byte[]> source, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException  {
+    SimpleWorkerContext res = new SimpleWorkerContext();
+    for (String name : source.keySet()) { 
+      try {
+        res.loadDefinitionItem(name, new ByteArrayInputStream(source.get(name)), loader);
+      } catch (Exception e) {
+        System.out.println("Error loading "+name+": "+e.getMessage());
+        throw new FHIRException("Error loading "+name+": "+e.getMessage(), e);
+      }
+    }
+    return res;
+  }
 	private void loadDefinitionItem(String name, InputStream stream, IContextResourceLoader loader) throws IOException, FHIRException {
     if (name.endsWith(".xml"))
       loadFromFile(stream, name, loader);
@@ -215,14 +249,19 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
         f = loader.loadBundle(stream, true);
       else {
         JsonParser json = new JsonParser();
-        f = (Bundle) json.parse(stream);
+        Resource r = json.parse(stream);
+        if (r instanceof Bundle)
+        f = (Bundle) r;
+        else {
+          f = new Bundle();
+          f.addEntry().setResource(r);
+        }
       }
     } catch (FHIRFormatError e1) {
       throw new org.hl7.fhir.exceptions.FHIRFormatError(e1.getMessage(), e1);
     }
     for (BundleEntryComponent e : f.getEntry()) {
-
+      if (e.getFullUrl() == null && logger != null) {
-      if (e.getFullUrl() == null) {
         logger.logDebugMessage(LogCategory.CONTEXT, "unidentified resource in " + name+" (no fullUrl)");
       }
       seeResource(e.getFullUrl(), e.getResource());
@@ -298,6 +337,15 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
 		loadFromStream(new CSFileInputStream(path), loader);
 	}
 
+	public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, String... types) throws FileNotFoundException, IOException, FHIRException {
+	  if (types.length == 0)
+	    types = new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire","ConceptMap","StructureMap", "NamingSystem"};
+	  for (String s : pi.listResources(types)) {
+      loadDefinitionItem(s, pi.load("package", s), loader);
+	  }
+	  version = pi.version();
+	}
+
   public void loadFromFile(String file, IContextResourceLoader loader) throws IOException, FHIRException {
     loadDefinitionItem(file, new CSFileInputStream(file), loader);
   }

org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/importers/Mimic14Importer.java

@@ -45,10 +45,12 @@ import org.fhir.ucum.UcumEssenceService;
 import org.fhir.ucum.UcumException;
 import org.fhir.ucum.UcumService;
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.dstu3.context.SimpleWorkerContext;
 import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
 import org.hl7.fhir.dstu3.formats.JsonParser;
 import org.hl7.fhir.dstu3.model.BaseDateTimeType;
 import org.hl7.fhir.dstu3.model.Bundle;
+import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.dstu3.model.Bundle.BundleType;
 import org.hl7.fhir.dstu3.model.CodeableConcept;
 import org.hl7.fhir.dstu3.model.Coding;
@@ -60,12 +62,14 @@ import org.hl7.fhir.dstu3.model.DocumentReference;
 import org.hl7.fhir.dstu3.model.Encounter;
 import org.hl7.fhir.dstu3.model.Enumerations.AdministrativeGender;
 import org.hl7.fhir.dstu3.model.Enumerations.DocumentReferenceStatus;
+import org.hl7.fhir.dstu3.model.ExpansionProfile;
 import org.hl7.fhir.dstu3.model.InstantType;
 import org.hl7.fhir.dstu3.model.MedicationRequest;
 import org.hl7.fhir.dstu3.model.MedicationRequest.MedicationRequestStatus;
 import org.hl7.fhir.dstu3.model.Observation;
 import org.hl7.fhir.dstu3.model.Observation.ObservationComponentComponent;
 import org.hl7.fhir.dstu3.model.Observation.ObservationStatus;
+import org.hl7.fhir.dstu3.model.Parameters;
 import org.hl7.fhir.dstu3.model.Patient;
 import org.hl7.fhir.dstu3.model.Period;
 import org.hl7.fhir.dstu3.model.Practitioner;
@@ -78,10 +82,15 @@ import org.hl7.fhir.dstu3.model.Quantity.QuantityComparator;
 import org.hl7.fhir.dstu3.model.Range;
 import org.hl7.fhir.dstu3.model.Reference;
 import org.hl7.fhir.dstu3.model.Type;
+import org.hl7.fhir.dstu3.utils.EOperationOutcome;
+import org.hl7.fhir.dstu3.utils.NarrativeGenerator;
+import org.hl7.fhir.dstu3.utils.ToolingExtensions;
 import org.hl7.fhir.utilities.CSVReader;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.ToolsVersion;
 
 public class Mimic14Importer {
 
@@ -117,8 +126,9 @@ public class Mimic14Importer {
   private IniFile ini;
 
   private UcumService ucum;
+  private SimpleWorkerContext context;
   
-  public static void main(String[] args) throws IOException, UcumException {
+  public static void main(String[] args) throws IOException, UcumException, FHIRException, EOperationOutcome {
     new Mimic14Importer().execute(args[0], args[1], args[2]);
   }
 
@@ -128,7 +138,12 @@ public class Mimic14Importer {
   }
 
 
-  private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException {
+  private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException, FHIRException, EOperationOutcome {
+    System.out.println("Loading Context");
+    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r3.core", "4.0.2"));
+    context.loadFromPackage(pcm.loadPackage("hl7.fhir.us.core", "1.1.0"), null, "StructureDefinition");
+    context.setExpansionProfile(new ExpansionProfile());
     System.out.println("Loading UCUM from "+ucumSrc);
   
    ucum = new UcumEssenceService(ucumSrc);
@@ -152,6 +167,15 @@ public class Mimic14Importer {
    
    System.out.println("saving");
    
+   NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context);
+   for (BundleEntryComponent be : patients.getEntry()) {
+     Patient p = (Patient) be.getResource();
+     gen.generate(p);
+   }
+   for (BundleEntryComponent be : encounters.getEntry()) {
+     Encounter p = (Encounter) be.getResource();
+     gen.generate(p);
+   }
    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "patients.json")), patients);    
    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "encounters.json")), encounters);
    
@@ -946,18 +970,23 @@ public class Mimic14Importer {
       }
       // ignore insurance
       if (csv.has("language")) {
+        pat.getCommunication().clear();
         pat.addCommunication().getLanguage().setText(csv.cell("language"));
       }
       if (csv.has("religion")) {
+        ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/StructureDefinition/patient-religion");
         pat.addExtension().setUrl("http://hl7.org/fhir/StructureDefinition/patient-religion").setValue(new CodeableConcept().setText(csv.cell("religion")));
       }
       if (csv.has("marital_status")) {
+        pat.getMaritalStatus().getCoding().clear();
         pat.getMaritalStatus().addCoding().setSystem("http://mimic.physionet.org/fhir/MaritalStatus").setCode(csv.cell("marital_status"));
       }
       if (csv.has("ethnicity")) {
+        ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity");
         pat.addExtension().setUrl("http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity").setValue(new CodeableConcept().setText(csv.cell("ethnicity")));
       }
       if (!csv.has("diagnosis")) {
+        enc.getReason().clear();
         enc.addReason().setText(csv.cell("diagnosis"));
       }
       if ("1".equals(csv.cell("hospital_expire_flag"))) {

org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/utils/BatchLoader.java

@@ -99,10 +99,12 @@ public class BatchLoader {
         bt.setType(BundleType.BATCH);     
         bt.setId(UUID.randomUUID().toString().toLowerCase());
         for (int i = cursor; i < Math.min(bnd.getEntry().size(), cursor+size); i++) {
-          BundleEntryComponent be = bt.addEntry();
+          if (i >=0 && i < bnd.getEntry().size()) {
-          be.setResource(bnd.getEntry().get(i).getResource());
+            BundleEntryComponent be = bt.addEntry();
-          be.getRequest().setMethod(HTTPVerb.PUT);
+            be.setResource(bnd.getEntry().get(i).getResource());
-          be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
+            be.getRequest().setMethod(HTTPVerb.PUT);
+            be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
+          }
         }
         System.out.print(f.getName()+" ("+cursor+"/"+bnd.getEntry().size()+"): ");
         ms = System.currentTimeMillis();

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/importers/Mimic14Importer.java

@@ -45,9 +45,12 @@ import org.fhir.ucum.UcumEssenceService;
 import org.fhir.ucum.UcumException;
 import org.fhir.ucum.UcumService;
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r4.context.IWorkerContext;
+import org.hl7.fhir.r4.context.SimpleWorkerContext;
 import org.hl7.fhir.r4.formats.IParser.OutputStyle;
 import org.hl7.fhir.r4.formats.JsonParser;
 import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.r4.model.Bundle.BundleType;
 import org.hl7.fhir.r4.model.CodeableConcept;
 import org.hl7.fhir.r4.model.Coding;
@@ -64,6 +67,7 @@ import org.hl7.fhir.r4.model.MedicationRequest.MedicationRequestStatus;
 import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Observation.ObservationComponentComponent;
 import org.hl7.fhir.r4.model.Observation.ObservationStatus;
+import org.hl7.fhir.r4.model.Parameters;
 import org.hl7.fhir.r4.model.Patient;
 import org.hl7.fhir.r4.model.Practitioner;
 import org.hl7.fhir.r4.model.PractitionerRole;
@@ -71,6 +75,10 @@ import org.hl7.fhir.r4.model.Procedure;
 import org.hl7.fhir.r4.model.Procedure.ProcedureStatus;
 import org.hl7.fhir.r4.model.Quantity;
 import org.hl7.fhir.r4.model.Quantity.QuantityComparator;
+import org.hl7.fhir.r4.utils.EOperationOutcome;
+import org.hl7.fhir.r4.utils.NarrativeGenerator;
+import org.hl7.fhir.r4.utils.NarrativeGenerator.ResourceContext;
+import org.hl7.fhir.r4.utils.ToolingExtensions;
 import org.hl7.fhir.r4.model.Range;
 import org.hl7.fhir.r4.model.Reference;
 import org.hl7.fhir.r4.model.Type;
@@ -78,6 +86,8 @@ import org.hl7.fhir.utilities.CSVReader;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.cache.PackageCacheManager;
+import org.hl7.fhir.utilities.cache.ToolsVersion;
 
 public class Mimic14Importer {
 
@@ -113,8 +123,9 @@ public class Mimic14Importer {
   private IniFile ini;
 
   private UcumService ucum;
+  private SimpleWorkerContext context;
   
-  public static void main(String[] args) throws IOException, UcumException {
+  public static void main(String[] args) throws IOException, UcumException, FHIRException, EOperationOutcome {
     new Mimic14Importer().execute(args[0], args[1], args[2]);
   }
 
@@ -124,7 +135,12 @@ public class Mimic14Importer {
   }
 
 
-  private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException {
+  private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException, FHIRException, EOperationOutcome {
+    System.out.println("Loading Context");
+    PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
+    context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r4.core", "4.0.1"));
+    context.loadFromPackage(pcm.loadPackage("hl7.fhir.us.core", "3.1.0"), null, "StructureDefinition");
+    context.setExpansionProfile(new Parameters());
     System.out.println("Loading UCUM from "+ucumSrc);
   
    ucum = new UcumEssenceService(ucumSrc);
@@ -148,6 +164,15 @@ public class Mimic14Importer {
    
    System.out.println("saving");
    
+   NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context);
+   for (BundleEntryComponent be : patients.getEntry()) {
+     Patient p = (Patient) be.getResource();
+     gen.generate(p, new HashSet<>());
+   }
+   for (BundleEntryComponent be : encounters.getEntry()) {
+     Encounter p = (Encounter) be.getResource();
+     gen.generate(p, new HashSet<>());
+   }
    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "patients.json")), patients);    
    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "encounters.json")), encounters);
    
@@ -924,7 +949,7 @@ public class Mimic14Importer {
     return new Quantity().setValue(new BigDecimal(cell));
   }
 
-  private Bundle processAdmissions(String src) throws FileNotFoundException, IOException {
+  private Bundle processAdmissions(String src) throws FileNotFoundException, IOException, FHIRException, EOperationOutcome {
     System.out.print("Processing Admissions... ");
     CSVReader csv = new CSVReader(new FileInputStream(src));
     Bundle bnd = new Bundle();
@@ -951,21 +976,27 @@ public class Mimic14Importer {
       }
       // ignore insurance
       if (csv.has("language")) {
+        pat.getCommunication().clear();
         pat.addCommunication().getLanguage().setText(csv.cell("language"));
       }
       if (csv.has("religion")) {
+        ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/StructureDefinition/patient-religion");
         pat.addExtension().setUrl("http://hl7.org/fhir/StructureDefinition/patient-religion").setValue(new CodeableConcept().setText(csv.cell("religion")));
       }
       if (csv.has("marital_status")) {
+        pat.getMaritalStatus().getCoding().clear();
         pat.getMaritalStatus().addCoding().setSystem("http://mimic.physionet.org/fhir/MaritalStatus").setCode(csv.cell("marital_status"));
       }
       if (csv.has("ethnicity")) {
+        ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity");
         pat.addExtension().setUrl("http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity").setValue(new CodeableConcept().setText(csv.cell("ethnicity")));
       }
       if (!csv.has("diagnosis")) {
+        enc.getReasonCode().clear();
         enc.addReasonCode().setText(csv.cell("diagnosis"));
       }
       if ("1".equals(csv.cell("hospital_expire_flag"))) {
+        enc.getHospitalization().getDischargeDisposition().getCoding().clear();
         enc.getHospitalization().getDischargeDisposition().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/discharge-disposition").setCode("exp");
       }
       bnd.addEntry().setResource(enc);