honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update Mimic importers

This commit is contained in:
Grahame Grieve 2019-11-15 08:16:12 +11:00
parent 1c4cceaf3c
commit df2a378eb0
4 changed files with 122 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/context/SimpleWorkerContext.java
View File

@ -79,6 +79,7 @@ import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.utilities.CSFileInputStream;
import org.hl7.fhir.utilities.OIDUtils;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.cache.NpmPackage;
import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
@ -129,6 +130,27 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
return res;
}
public static SimpleWorkerContext fromPackage(NpmPackage pi, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException {
SimpleWorkerContext res = new SimpleWorkerContext();
res.setAllowLoadingDuplicates(allowDuplicates);
res.loadFromPackage(pi, null);
return res;
}
public static SimpleWorkerContext fromPackage(NpmPackage pi) throws FileNotFoundException, IOException, FHIRException {
SimpleWorkerContext res = new SimpleWorkerContext();
res.loadFromPackage(pi, null);
return res;
}
public static SimpleWorkerContext fromPackage(NpmPackage pi, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException {
SimpleWorkerContext res = new SimpleWorkerContext();
res.setAllowLoadingDuplicates(true);
res.version = pi.getNpm().get("version").getAsString();
res.loadFromPackage(pi, loader);
return res;
}
public static SimpleWorkerContext fromPack(String path, boolean allowDuplicates) throws FileNotFoundException, IOException, FHIRException {
SimpleWorkerContext res = new SimpleWorkerContext();
res.allowLoadingDuplicates = allowDuplicates;
@ -163,6 +185,18 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
return res;
}
public static SimpleWorkerContext fromDefinitions(Map<String, byte[]> source, IContextResourceLoader loader) throws FileNotFoundException, IOException, FHIRException {
SimpleWorkerContext res = new SimpleWorkerContext();
for (String name : source.keySet()) {
try {
res.loadDefinitionItem(name, new ByteArrayInputStream(source.get(name)), loader);
} catch (Exception e) {
System.out.println("Error loading "+name+": "+e.getMessage());
throw new FHIRException("Error loading "+name+": "+e.getMessage(), e);
}
}
return res;
}
private void loadDefinitionItem(String name, InputStream stream, IContextResourceLoader loader) throws IOException, FHIRException {
if (name.endsWith(".xml"))
loadFromFile(stream, name, loader);
@ -215,14 +249,19 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
f = loader.loadBundle(stream, true);
else {
JsonParser json = new JsonParser();
f = (Bundle) json.parse(stream);
Resource r = json.parse(stream);
if (r instanceof Bundle)
f = (Bundle) r;
else {
f = new Bundle();
f.addEntry().setResource(r);
}
}
} catch (FHIRFormatError e1) {
throw new org.hl7.fhir.exceptions.FHIRFormatError(e1.getMessage(), e1);
}
for (BundleEntryComponent e : f.getEntry()) {
if (e.getFullUrl() == null) {
if (e.getFullUrl() == null && logger != null) {
logger.logDebugMessage(LogCategory.CONTEXT, "unidentified resource in " + name+" (no fullUrl)");
}
seeResource(e.getFullUrl(), e.getResource());
@ -298,6 +337,15 @@ public class SimpleWorkerContext extends BaseWorkerContext implements IWorkerCon
loadFromStream(new CSFileInputStream(path), loader);
}
public void loadFromPackage(NpmPackage pi, IContextResourceLoader loader, String... types) throws FileNotFoundException, IOException, FHIRException {
if (types.length == 0)
types = new String[] { "StructureDefinition", "ValueSet", "CodeSystem", "SearchParameter", "OperationDefinition", "Questionnaire","ConceptMap","StructureMap", "NamingSystem"};
for (String s : pi.listResources(types)) {
loadDefinitionItem(s, pi.load("package", s), loader);
}
version = pi.version();
}
public void loadFromFile(String file, IContextResourceLoader loader) throws IOException, FHIRException {
loadDefinitionItem(file, new CSFileInputStream(file), loader);
}

33
org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/importers/Mimic14Importer.java
View File

@ -45,10 +45,12 @@ import org.fhir.ucum.UcumEssenceService;
import org.fhir.ucum.UcumException;
import org.fhir.ucum.UcumService;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.dstu3.context.SimpleWorkerContext;
import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
import org.hl7.fhir.dstu3.formats.JsonParser;
import org.hl7.fhir.dstu3.model.BaseDateTimeType;
import org.hl7.fhir.dstu3.model.Bundle;
import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.dstu3.model.Bundle.BundleType;
import org.hl7.fhir.dstu3.model.CodeableConcept;
import org.hl7.fhir.dstu3.model.Coding;
@ -60,12 +62,14 @@ import org.hl7.fhir.dstu3.model.DocumentReference;
import org.hl7.fhir.dstu3.model.Encounter;
import org.hl7.fhir.dstu3.model.Enumerations.AdministrativeGender;
import org.hl7.fhir.dstu3.model.Enumerations.DocumentReferenceStatus;
import org.hl7.fhir.dstu3.model.ExpansionProfile;
import org.hl7.fhir.dstu3.model.InstantType;
import org.hl7.fhir.dstu3.model.MedicationRequest;
import org.hl7.fhir.dstu3.model.MedicationRequest.MedicationRequestStatus;
import org.hl7.fhir.dstu3.model.Observation;
import org.hl7.fhir.dstu3.model.Observation.ObservationComponentComponent;
import org.hl7.fhir.dstu3.model.Observation.ObservationStatus;
import org.hl7.fhir.dstu3.model.Parameters;
import org.hl7.fhir.dstu3.model.Patient;
import org.hl7.fhir.dstu3.model.Period;
import org.hl7.fhir.dstu3.model.Practitioner;
@ -78,10 +82,15 @@ import org.hl7.fhir.dstu3.model.Quantity.QuantityComparator;
import org.hl7.fhir.dstu3.model.Range;
import org.hl7.fhir.dstu3.model.Reference;
import org.hl7.fhir.dstu3.model.Type;
import org.hl7.fhir.dstu3.utils.EOperationOutcome;
import org.hl7.fhir.dstu3.utils.NarrativeGenerator;
import org.hl7.fhir.dstu3.utils.ToolingExtensions;
import org.hl7.fhir.utilities.CSVReader;
import org.hl7.fhir.utilities.IniFile;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.cache.PackageCacheManager;
import org.hl7.fhir.utilities.cache.ToolsVersion;
public class Mimic14Importer {
@ -117,8 +126,9 @@ public class Mimic14Importer {
private IniFile ini;
private UcumService ucum;
private SimpleWorkerContext context;
public static void main(String[] args) throws IOException, UcumException {
public static void main(String[] args) throws IOException, UcumException, FHIRException, EOperationOutcome {
new Mimic14Importer().execute(args[0], args[1], args[2]);
}
@ -128,7 +138,12 @@ public class Mimic14Importer {
}
private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException {
private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException, FHIRException, EOperationOutcome {
System.out.println("Loading Context");
PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r3.core", "4.0.2"));
context.loadFromPackage(pcm.loadPackage("hl7.fhir.us.core", "1.1.0"), null, "StructureDefinition");
context.setExpansionProfile(new ExpansionProfile());
System.out.println("Loading UCUM from "+ucumSrc);
ucum = new UcumEssenceService(ucumSrc);
@ -152,6 +167,15 @@ public class Mimic14Importer {
System.out.println("saving");
NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context);
for (BundleEntryComponent be : patients.getEntry()) {
Patient p = (Patient) be.getResource();
gen.generate(p);
}
for (BundleEntryComponent be : encounters.getEntry()) {
Encounter p = (Encounter) be.getResource();
gen.generate(p);
}
new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "patients.json")), patients);
new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "encounters.json")), encounters);
@ -946,18 +970,23 @@ public class Mimic14Importer {
}
// ignore insurance
if (csv.has("language")) {
pat.getCommunication().clear();
pat.addCommunication().getLanguage().setText(csv.cell("language"));
}
if (csv.has("religion")) {
ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/StructureDefinition/patient-religion");
pat.addExtension().setUrl("http://hl7.org/fhir/StructureDefinition/patient-religion").setValue(new CodeableConcept().setText(csv.cell("religion")));
}
if (csv.has("marital_status")) {
pat.getMaritalStatus().getCoding().clear();
pat.getMaritalStatus().addCoding().setSystem("http://mimic.physionet.org/fhir/MaritalStatus").setCode(csv.cell("marital_status"));
}
if (csv.has("ethnicity")) {
ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity");
pat.addExtension().setUrl("http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity").setValue(new CodeableConcept().setText(csv.cell("ethnicity")));
}
if (!csv.has("diagnosis")) {
enc.getReason().clear();
enc.addReason().setText(csv.cell("diagnosis"));
}
if ("1".equals(csv.cell("hospital_expire_flag"))) {

2
org.hl7.fhir.dstu3/src/main/java/org/hl7/fhir/dstu3/utils/BatchLoader.java
View File

@ -99,11 +99,13 @@ public class BatchLoader {
bt.setType(BundleType.BATCH);
bt.setId(UUID.randomUUID().toString().toLowerCase());
for (int i = cursor; i < Math.min(bnd.getEntry().size(), cursor+size); i++) {
if (i >=0 && i < bnd.getEntry().size()) {
BundleEntryComponent be = bt.addEntry();
be.setResource(bnd.getEntry().get(i).getResource());
be.getRequest().setMethod(HTTPVerb.PUT);
be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
}
}
System.out.print(f.getName()+" ("+cursor+"/"+bnd.getEntry().size()+"): ");
ms = System.currentTimeMillis();
Bundle resp = client.transaction(bt);

37
org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/importers/Mimic14Importer.java
View File

@ -45,9 +45,12 @@ import org.fhir.ucum.UcumEssenceService;
import org.fhir.ucum.UcumException;
import org.fhir.ucum.UcumService;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.r4.context.IWorkerContext;
import org.hl7.fhir.r4.context.SimpleWorkerContext;
import org.hl7.fhir.r4.formats.IParser.OutputStyle;
import org.hl7.fhir.r4.formats.JsonParser;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.r4.model.Bundle.BundleType;
import org.hl7.fhir.r4.model.CodeableConcept;
import org.hl7.fhir.r4.model.Coding;
@ -64,6 +67,7 @@ import org.hl7.fhir.r4.model.MedicationRequest.MedicationRequestStatus;
import org.hl7.fhir.r4.model.Observation;
import org.hl7.fhir.r4.model.Observation.ObservationComponentComponent;
import org.hl7.fhir.r4.model.Observation.ObservationStatus;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.Patient;
import org.hl7.fhir.r4.model.Practitioner;
import org.hl7.fhir.r4.model.PractitionerRole;
@ -71,6 +75,10 @@ import org.hl7.fhir.r4.model.Procedure;
import org.hl7.fhir.r4.model.Procedure.ProcedureStatus;
import org.hl7.fhir.r4.model.Quantity;
import org.hl7.fhir.r4.model.Quantity.QuantityComparator;
import org.hl7.fhir.r4.utils.EOperationOutcome;
import org.hl7.fhir.r4.utils.NarrativeGenerator;
import org.hl7.fhir.r4.utils.NarrativeGenerator.ResourceContext;
import org.hl7.fhir.r4.utils.ToolingExtensions;
import org.hl7.fhir.r4.model.Range;
import org.hl7.fhir.r4.model.Reference;
import org.hl7.fhir.r4.model.Type;
@ -78,6 +86,8 @@ import org.hl7.fhir.utilities.CSVReader;
import org.hl7.fhir.utilities.IniFile;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.cache.PackageCacheManager;
import org.hl7.fhir.utilities.cache.ToolsVersion;
public class Mimic14Importer {
@ -113,8 +123,9 @@ public class Mimic14Importer {
private IniFile ini;
private UcumService ucum;
private SimpleWorkerContext context;
public static void main(String[] args) throws IOException, UcumException {
public static void main(String[] args) throws IOException, UcumException, FHIRException, EOperationOutcome {
new Mimic14Importer().execute(args[0], args[1], args[2]);
}
@ -124,7 +135,12 @@ public class Mimic14Importer {
}
private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException {
private void execute(String src, String dest, String ucumSrc) throws IOException, UcumException, FHIRException, EOperationOutcome {
System.out.println("Loading Context");
PackageCacheManager pcm = new PackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
context = SimpleWorkerContext.fromPackage(pcm.loadPackage("hl7.fhir.r4.core", "4.0.1"));
context.loadFromPackage(pcm.loadPackage("hl7.fhir.us.core", "3.1.0"), null, "StructureDefinition");
context.setExpansionProfile(new Parameters());
System.out.println("Loading UCUM from "+ucumSrc);
ucum = new UcumEssenceService(ucumSrc);
@ -148,6 +164,15 @@ public class Mimic14Importer {
System.out.println("saving");
NarrativeGenerator gen = new NarrativeGenerator("", "http://hl7.org/fhir", context);
for (BundleEntryComponent be : patients.getEntry()) {
Patient p = (Patient) be.getResource();
gen.generate(p, new HashSet<>());
}
for (BundleEntryComponent be : encounters.getEntry()) {
Encounter p = (Encounter) be.getResource();
gen.generate(p, new HashSet<>());
}
new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "patients.json")), patients);
new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "encounters.json")), encounters);
@ -924,7 +949,7 @@ public class Mimic14Importer {
return new Quantity().setValue(new BigDecimal(cell));
}
private Bundle processAdmissions(String src) throws FileNotFoundException, IOException {
private Bundle processAdmissions(String src) throws FileNotFoundException, IOException, FHIRException, EOperationOutcome {
System.out.print("Processing Admissions... ");
CSVReader csv = new CSVReader(new FileInputStream(src));
Bundle bnd = new Bundle();
@ -951,21 +976,27 @@ public class Mimic14Importer {
}
// ignore insurance
if (csv.has("language")) {
pat.getCommunication().clear();
pat.addCommunication().getLanguage().setText(csv.cell("language"));
}
if (csv.has("religion")) {
ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/StructureDefinition/patient-religion");
pat.addExtension().setUrl("http://hl7.org/fhir/StructureDefinition/patient-religion").setValue(new CodeableConcept().setText(csv.cell("religion")));
}
if (csv.has("marital_status")) {
pat.getMaritalStatus().getCoding().clear();
pat.getMaritalStatus().addCoding().setSystem("http://mimic.physionet.org/fhir/MaritalStatus").setCode(csv.cell("marital_status"));
}
if (csv.has("ethnicity")) {
ToolingExtensions.removeExtension(pat, "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity");
pat.addExtension().setUrl("http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity").setValue(new CodeableConcept().setText(csv.cell("ethnicity")));
}
if (!csv.has("diagnosis")) {
enc.getReasonCode().clear();
enc.addReasonCode().setText(csv.cell("diagnosis"));
}
if ("1".equals(csv.cell("hospital_expire_flag"))) {
enc.getHospitalization().getDischargeDisposition().getCoding().clear();
enc.getHospitalization().getDischargeDisposition().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/discharge-disposition").setCode("exp");
}
bnd.addEntry().setResource(enc);