From df2a378eb0754e41238ead22c1c7ce704f705a82 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Fri, 15 Nov 2019 08:16:12 +1100 Subject: [PATCH] update Mimic importers --- .../dstu3/context/SimpleWorkerContext.java | 54 +++++++++++++++++-- .../fhir/dstu3/importers/Mimic14Importer.java | 33 +++++++++++- .../org/hl7/fhir/dstu3/utils/BatchLoader.java | 10 ++-- .../fhir/r4/importers/Mimic14Importer.java | 37 +++++++++++-- 4 files changed, 122 insertions(+), 12 deletions(-) diff --git a/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/context/SimpleWorkerContext.java b/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/context/SimpleWorkerContext.java index 3e8fb1f2c..3da505320 100644 --- a/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/context/SimpleWorkerContext.java +++ b/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/context/SimpleWorkerContext.java @@ -79,6 +79,7 @@ import org.hl7.fhir.exceptions.FHIRFormatError; import org.hl7.fhir.utilities.CSFileInputStream; import org.hl7.fhir.utilities.OIDUtils; import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.cache.NpmPackage; import org.hl7.fhir.utilities.validation.ValidationMessage; import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType; import org.hl7.fhir.utilities.validation.ValidationMessage.Source; @@ -129,6 +130,27 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon return res; } + public static SimpleWorkerContext fromPackage(NpmPackage pi, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException { + SimpleWorkerContext res = new SimpleWorkerContext(); + res.setAllowLoadingDuplicates(allowDuplicates); + res.loadFromPackage(pi, null); + return res; + } + + public static SimpleWorkerContext fromPackage(NpmPackage pi) throws FileNotFoundException, IOException, FHIRException { + SimpleWorkerContext res = new SimpleWorkerContext(); + res.loadFromPackage(pi, null); + return res; + } + + public static SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException { + SimpleWorkerContext res = new SimpleWorkerContext(); + res.setAllowLoadingDuplicates(true); + res.version = pi.getNpm().get("version").getAsString(); + res.loadFromPackage(pi, loader); + return res; + } + public static SimpleWorkerContext fromPack(String path, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException { SimpleWorkerContext res = new SimpleWorkerContext(); res.allowLoadingDuplicates = allowDuplicates; @@ -163,6 +185,18 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon return res; } + public static SimpleWorkerContext fromDefinitions(Map source, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException { + SimpleWorkerContext res = new SimpleWorkerContext(); + for (String name : source.keySet()) { + try { + res.loadDefinitionItem(name, new ByteArrayInputStream(source.get(name)), loader); + } catch (Exception e) { + System.out.println("Error loading "+name+": "+e.getMessage()); + throw new FHIRException("Error loading "+name+": "+e.getMessage(), e); + } + } + return res; + } private void loadDefinitionItem(String name, InputStream stream, IContextResourceLoader loader) throws IOException, FHIRException { if (name.endsWith(".xml")) loadFromFile(stream, name, loader); @@ -215,14 +249,19 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon f = loader.loadBundle(stream, true); else { JsonParser json = new JsonParser(); - f = (Bundle) json.parse(stream); + Resource r = json.parse(stream); + if (r instanceof Bundle) + f = (Bundle) r; + else { + f = new Bundle(); + f.addEntry().setResource(r); + } } } catch (FHIRFormatError e1) { throw new org.hl7.fhir.exceptions.FHIRFormatError(e1.getMessage(), e1); } for (BundleEntryComponent e : f.getEntry()) { - - if (e.getFullUrl() == null) { + if (e.getFullUrl() == null && logger != null) { logger.logDebugMessage(LogCategory.CONTEXT, "unidentified resource in " + name+" (no fullUrl)"); } seeResource(e.getFullUrl(), e.getResource()); @@ -298,6 +337,15 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon loadFromStream(new CSFileInputStream(path), loader); } + public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, String... types) throws FileNotFoundException, IOException, FHIRException { + if (types.length == 0) + types = new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire","ConceptMap","StructureMap", "NamingSystem"}; + for (String s : pi.listResources(types)) { + loadDefinitionItem(s, pi.load("package", s), loader); + } + version = pi.version(); + } + public void loadFromFile(String file, IContextResourceLoader loader) throws IOException, FHIRException { loadDefinitionItem(file, new CSFileInputStream(file), loader); } diff --git a/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/importers/Mimic14Importer.java b/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/importers/Mimic14Importer.java index 69ad98622..aee161d77 100644 --- a/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/importers/Mimic14Importer.java +++ b/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/importers/Mimic14Importer.java @@ -45,10 +45,12 @@ import org.fhir.ucum.UcumEssenceService; import org.fhir.ucum.UcumException; import org.fhir.ucum.UcumService; import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.dstu3.context.SimpleWorkerContext; import org.hl7.fhir.dstu3.formats.IParser.OutputStyle; import org.hl7.fhir.dstu3.formats.JsonParser; import org.hl7.fhir.dstu3.model.BaseDateTimeType; import org.hl7.fhir.dstu3.model.Bundle; +import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent; import org.hl7.fhir.dstu3.model.Bundle.BundleType; import org.hl7.fhir.dstu3.model.CodeableConcept; import org.hl7.fhir.dstu3.model.Coding; @@ -60,12 +62,14 @@ import org.hl7.fhir.dstu3.model.DocumentReference; import org.hl7.fhir.dstu3.model.Encounter; import org.hl7.fhir.dstu3.model.Enumerations.AdministrativeGender; import org.hl7.fhir.dstu3.model.Enumerations.DocumentReferenceStatus; +import org.hl7.fhir.dstu3.model.ExpansionProfile; import org.hl7.fhir.dstu3.model.InstantType; import org.hl7.fhir.dstu3.model.MedicationRequest; import org.hl7.fhir.dstu3.model.MedicationRequest.MedicationRequestStatus; import org.hl7.fhir.dstu3.model.Observation; import org.hl7.fhir.dstu3.model.Observation.ObservationComponentComponent; import org.hl7.fhir.dstu3.model.Observation.ObservationStatus; +import org.hl7.fhir.dstu3.model.Parameters; import org.hl7.fhir.dstu3.model.Patient; import org.hl7.fhir.dstu3.model.Period; import org.hl7.fhir.dstu3.model.Practitioner; @@ -78,10 +82,15 @@ import org.hl7.fhir.dstu3.model.Quantity.QuantityComparator; import org.hl7.fhir.dstu3.model.Range; import org.hl7.fhir.dstu3.model.Reference; import org.hl7.fhir.dstu3.model.Type; +import org.hl7.fhir.dstu3.utils.EOperationOutcome; +import org.hl7.fhir.dstu3.utils.NarrativeGenerator; +import org.hl7.fhir.dstu3.utils.ToolingExtensions; import org.hl7.fhir.utilities.CSVReader; import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.ToolsVersion; public class Mimic14Importer { @@ -117,8 +126,9 @@ public class Mimic14Importer { private IniFile ini; private UcumService ucum; + private SimpleWorkerContext context; - public static void main(String[] args) throws IOException, UcumException { + public static void main(String[] args) throws IOException, UcumException, FHIRException, EOperationOutcome { new Mimic14Importer().execute(args[0], args[1], args[2]); } @@ -128,7 +138,12 @@ public class Mimic14Importer { } - private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException { + private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException, FHIRException, EOperationOutcome { + System.out.println("Loading Context"); + PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r3.core", "4.0.2")); + context.loadFromPackage(pcm.loadPackage("hl7.fhir.us.core", "1.1.0"), null, "StructureDefinition"); + context.setExpansionProfile(new ExpansionProfile()); System.out.println("Loading UCUM from "+ucumSrc); ucum = new UcumEssenceService(ucumSrc); @@ -152,6 +167,15 @@ public class Mimic14Importer { System.out.println("saving"); + NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context); + for (BundleEntryComponent be : patients.getEntry()) { + Patient p = (Patient) be.getResource(); + gen.generate(p); + } + for (BundleEntryComponent be : encounters.getEntry()) { + Encounter p = (Encounter) be.getResource(); + gen.generate(p); + } new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "patients.json")), patients); new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "encounters.json")), encounters); @@ -946,18 +970,23 @@ public class Mimic14Importer { } // ignore insurance if (csv.has("language")) { + pat.getCommunication().clear(); pat.addCommunication().getLanguage().setText(csv.cell("language")); } if (csv.has("religion")) { + ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/StructureDefinition/patient-religion"); pat.addExtension().setUrl("http://hl7.org/fhir/StructureDefinition/patient-religion").setValue(new CodeableConcept().setText(csv.cell("religion"))); } if (csv.has("marital_status")) { + pat.getMaritalStatus().getCoding().clear(); pat.getMaritalStatus().addCoding().setSystem("http://mimic.physionet.org/fhir/MaritalStatus").setCode(csv.cell("marital_status")); } if (csv.has("ethnicity")) { + ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity"); pat.addExtension().setUrl("http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity").setValue(new CodeableConcept().setText(csv.cell("ethnicity"))); } if (!csv.has("diagnosis")) { + enc.getReason().clear(); enc.addReason().setText(csv.cell("diagnosis")); } if ("1".equals(csv.cell("hospital_expire_flag"))) { diff --git a/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/utils/BatchLoader.java b/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/utils/BatchLoader.java index a1d2ce848..d7cb68603 100644 --- a/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/utils/BatchLoader.java +++ b/org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/utils/BatchLoader.java @@ -99,10 +99,12 @@ public class BatchLoader { bt.setType(BundleType.BATCH); bt.setId(UUID.randomUUID().toString().toLowerCase()); for (int i = cursor; i < Math.min(bnd.getEntry().size(), cursor+size); i++) { - BundleEntryComponent be = bt.addEntry(); - be.setResource(bnd.getEntry().get(i).getResource()); - be.getRequest().setMethod(HTTPVerb.PUT); - be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId()); + if (i >=0 && i < bnd.getEntry().size()) { + BundleEntryComponent be = bt.addEntry(); + be.setResource(bnd.getEntry().get(i).getResource()); + be.getRequest().setMethod(HTTPVerb.PUT); + be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId()); + } } System.out.print(f.getName()+" ("+cursor+"/"+bnd.getEntry().size()+"): "); ms = System.currentTimeMillis(); diff --git a/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/importers/Mimic14Importer.java b/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/importers/Mimic14Importer.java index 1b0d43e0e..a0f39d170 100644 --- a/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/importers/Mimic14Importer.java +++ b/org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/importers/Mimic14Importer.java @@ -45,9 +45,12 @@ import org.fhir.ucum.UcumEssenceService; import org.fhir.ucum.UcumException; import org.fhir.ucum.UcumService; import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.r4.context.IWorkerContext; +import org.hl7.fhir.r4.context.SimpleWorkerContext; import org.hl7.fhir.r4.formats.IParser.OutputStyle; import org.hl7.fhir.r4.formats.JsonParser; import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; import org.hl7.fhir.r4.model.Bundle.BundleType; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; @@ -64,6 +67,7 @@ import org.hl7.fhir.r4.model.MedicationRequest.MedicationRequestStatus; import org.hl7.fhir.r4.model.Observation; import org.hl7.fhir.r4.model.Observation.ObservationComponentComponent; import org.hl7.fhir.r4.model.Observation.ObservationStatus; +import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Practitioner; import org.hl7.fhir.r4.model.PractitionerRole; @@ -71,6 +75,10 @@ import org.hl7.fhir.r4.model.Procedure; import org.hl7.fhir.r4.model.Procedure.ProcedureStatus; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; +import org.hl7.fhir.r4.utils.EOperationOutcome; +import org.hl7.fhir.r4.utils.NarrativeGenerator; +import org.hl7.fhir.r4.utils.NarrativeGenerator.ResourceContext; +import org.hl7.fhir.r4.utils.ToolingExtensions; import org.hl7.fhir.r4.model.Range; import org.hl7.fhir.r4.model.Reference; import org.hl7.fhir.r4.model.Type; @@ -78,6 +86,8 @@ import org.hl7.fhir.utilities.CSVReader; import org.hl7.fhir.utilities.IniFile; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.cache.PackageCacheManager; +import org.hl7.fhir.utilities.cache.ToolsVersion; public class Mimic14Importer { @@ -113,8 +123,9 @@ public class Mimic14Importer { private IniFile ini; private UcumService ucum; + private SimpleWorkerContext context; - public static void main(String[] args) throws IOException, UcumException { + public static void main(String[] args) throws IOException, UcumException, FHIRException, EOperationOutcome { new Mimic14Importer().execute(args[0], args[1], args[2]); } @@ -124,7 +135,12 @@ public class Mimic14Importer { } - private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException { + private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException, FHIRException, EOperationOutcome { + System.out.println("Loading Context"); + PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION); + context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r4.core", "4.0.1")); + context.loadFromPackage(pcm.loadPackage("hl7.fhir.us.core", "3.1.0"), null, "StructureDefinition"); + context.setExpansionProfile(new Parameters()); System.out.println("Loading UCUM from "+ucumSrc); ucum = new UcumEssenceService(ucumSrc); @@ -148,6 +164,15 @@ public class Mimic14Importer { System.out.println("saving"); + NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context); + for (BundleEntryComponent be : patients.getEntry()) { + Patient p = (Patient) be.getResource(); + gen.generate(p, new HashSet<>()); + } + for (BundleEntryComponent be : encounters.getEntry()) { + Encounter p = (Encounter) be.getResource(); + gen.generate(p, new HashSet<>()); + } new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "patients.json")), patients); new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "encounters.json")), encounters); @@ -924,7 +949,7 @@ public class Mimic14Importer { return new Quantity().setValue(new BigDecimal(cell)); } - private Bundle processAdmissions(String src) throws FileNotFoundException, IOException { + private Bundle processAdmissions(String src) throws FileNotFoundException, IOException, FHIRException, EOperationOutcome { System.out.print("Processing Admissions... "); CSVReader csv = new CSVReader(new FileInputStream(src)); Bundle bnd = new Bundle(); @@ -951,21 +976,27 @@ public class Mimic14Importer { } // ignore insurance if (csv.has("language")) { + pat.getCommunication().clear(); pat.addCommunication().getLanguage().setText(csv.cell("language")); } if (csv.has("religion")) { + ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/StructureDefinition/patient-religion"); pat.addExtension().setUrl("http://hl7.org/fhir/StructureDefinition/patient-religion").setValue(new CodeableConcept().setText(csv.cell("religion"))); } if (csv.has("marital_status")) { + pat.getMaritalStatus().getCoding().clear(); pat.getMaritalStatus().addCoding().setSystem("http://mimic.physionet.org/fhir/MaritalStatus").setCode(csv.cell("marital_status")); } if (csv.has("ethnicity")) { + ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity"); pat.addExtension().setUrl("http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity").setValue(new CodeableConcept().setText(csv.cell("ethnicity"))); } if (!csv.has("diagnosis")) { + enc.getReasonCode().clear(); enc.addReasonCode().setText(csv.cell("diagnosis")); } if ("1".equals(csv.cell("hospital_expire_flag"))) { + enc.getHospitalization().getDischargeDisposition().getCoding().clear(); enc.getHospitalization().getDischargeDisposition().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/discharge-disposition").setCode("exp"); } bnd.addEntry().setResource(enc);