From fe7efcf5a2eb8d9386459f3740a843301ec78923 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sat, 22 Aug 2020 08:09:50 +1000 Subject: [PATCH] Fix bugs in PHINVads importer --- .../convertors/misc/PhinVadsImporter.java | 19 +++++++++--- .../org/hl7/fhir/utilities/CSVReader.java | 29 +++++++++++++++---- .../fhir/utilities/cache/PackageHacker.java | 14 ++++----- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/PhinVadsImporter.java b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/PhinVadsImporter.java index e106bb003..03b63fb99 100644 --- a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/PhinVadsImporter.java +++ b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/PhinVadsImporter.java @@ -26,6 +26,7 @@ import org.hl7.fhir.utilities.cache.ToolsVersion; public class PhinVadsImporter { public static void main(String[] args) throws FileNotFoundException, FHIRException, IOException, ParseException { +// new PhinVadsImporter().importValueSet(TextFile.fileToBytes("C:\\work\\org.hl7.fhir\\packages\\us.cdc.phinvads-source\\source\\PHVS_BirthDefectsLateralityatDiagnosis_HL7_V1.txt")); PhinVadsImporter self = new PhinVadsImporter(); self.init(); self.process(args[0], args[1]); @@ -61,17 +62,20 @@ public class PhinVadsImporter { private ValueSet importValueSet(byte[] source) throws FHIRException, IOException, ParseException { // first thing do is split into 2 List parts = Utilities.splitBytes(source, "\r\n\r\n".getBytes()); - if (parts.size() != 2) { + if (parts.size() < 2) { + TextFile.bytesToFile(source, "c:\\temp\\phinvads.txt"); throw new FHIRException("Unable to parse phinvads value set: "+parts.size()+" parts found"); } CSVReader rdr = new CSVReader(new ByteArrayInputStream(parts.get(0))); rdr.setDelimiter('\t'); + rdr.setMultiline(true); rdr.readHeaders(); rdr.line(); ValueSet vs = new ValueSet(); vs.setId(rdr.cell("Value Set OID")); vs.setUrl("https://phinvads.cdc.gov/fhir/ValueSet/"+vs.getId()); + vs.getMeta().setSource("https://phinvads.cdc.gov/vads/ViewValueSet.action?oid="+vs.getId()); vs.setVersion(rdr.cell("Value Set Version")); vs.setTitle(rdr.cell("Value Set Name")); vs.setName(rdr.cell("Value Set Code")); @@ -83,7 +87,8 @@ public class PhinVadsImporter { vs.setDate(new SimpleDateFormat("mm/dd/yyyy").parse(rdr.cell("VS Last Updated Date"))); } - rdr = new CSVReader(new ByteArrayInputStream(parts.get(1))); + rdr = new CSVReader(new ByteArrayInputStream(parts.get(parts.size()-1))); + rdr.setMultiline(true); rdr.setDelimiter('\t'); rdr.readHeaders(); while (rdr.line()) { @@ -103,8 +108,14 @@ public class PhinVadsImporter { private ConceptSetComponent getInclude(ValueSet vs, String url, String csver) { for (ConceptSetComponent t : vs.getCompose().getInclude()) { - if (t.getSystem().equals(url) && t.getVersion().equals(csver)) { - return t; + if (csver == null) { + if (t.getSystem().equals(url) && !t.hasVersion()) { + return t; + } + } else { + if (t.getSystem().equals(url) && t.hasVersion() && t.getVersion().equals(csver)) { + return t; + } } } ConceptSetComponent c = vs.getCompose().addInclude(); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/CSVReader.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/CSVReader.java index 49472e3ef..60009a350 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/CSVReader.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/CSVReader.java @@ -61,11 +61,11 @@ public class CSVReader extends InputStreamReader { private String[] cols; private String[] cells; private char delimiter = ','; + private boolean multiline; public void readHeaders() throws IOException, FHIRException { cols = parseLine(); } - public boolean line() throws IOException, FHIRException { if (ready()) { @@ -128,7 +128,7 @@ public class CSVReader extends InputStreamReader { /** - * Split one line in a CSV file into its rows. Comma's appearing in double quoted strings will + * Split one line in a CSV file into its cells. Comma's appearing in double quoted strings will * not be seen as a separator. * @return * @throws IOException @@ -140,7 +140,7 @@ public class CSVReader extends InputStreamReader { StringBuilder b = new StringBuilder(); boolean inQuote = false; - while (ready() && (inQuote || (peek() != '\r' && peek() != '\n'))) { + while (more() && !finished(inQuote, res.size())) { char c = peek(); next(); if (c == '"') { @@ -166,10 +166,21 @@ public class CSVReader extends InputStreamReader { String[] r = new String[] {}; r = res.toArray(r); return r; - } - private int state = 0; + private boolean more() throws IOException { + return state == 1 || ready(); + } + + private boolean finished(boolean inQuote, int size) throws FHIRException, IOException { + if (multiline && cols != null) { + return size == cols.length || (size == cols.length - 1 && !(inQuote || (peek() != '\r' && peek() != '\n'))); + } else { + return !(inQuote || (peek() != '\r' && peek() != '\n')); + } + } + + private int state = 0; private char pc; private char peek() throws FHIRException, IOException @@ -219,5 +230,13 @@ public class CSVReader extends InputStreamReader { this.delimiter = delimiter; } + public boolean isMultiline() { + return multiline; + } + + public void setMultiline(boolean multiline) { + this.multiline = multiline; + } + } \ No newline at end of file diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageHacker.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageHacker.java index 62ad4dd1c..f855b8f79 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageHacker.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/cache/PackageHacker.java @@ -25,7 +25,7 @@ import com.google.gson.JsonObject; public class PackageHacker { public static void main(String[] args) throws FileNotFoundException, IOException { - new PackageHacker().edit("M:\\web\\hl7.org\\fhir\\us\\davinci-pdex-plan-net\\2020Feb\\package.tgz"); + new PackageHacker().edit("M:\\web\\hl7.org\\fhir\\2020Sep\\hl7.fhir.r5.expansions.tgz"); } private void edit(String name) throws FileNotFoundException, IOException { @@ -56,11 +56,11 @@ public class PackageHacker { } private void change(JsonObject npm, Map content) throws FileNotFoundException, IOException { -// fixVersions(npm); + fixVersions(npm); // npm.remove("url"); // npm.addProperty("url", url); -// npm.remove("version"); -// npm.addProperty("version", "1.0.0"); + npm.remove("version"); + npm.addProperty("version", "4.5.0"); // npm.remove("canonical"); // npm.addProperty("canonical", "http://hl7.org/fhir/us/davinci-drug-formulary"); //// npm.remove("description"); @@ -80,11 +80,7 @@ public class PackageHacker { npm.remove("fhirVersions"); JsonArray a = new JsonArray(); npm.add("fhirVersions", a); - a.add("4.2.0"); - npm.remove("fhir-version-list"); - a = new JsonArray(); - npm.add("fhir-version-list", a); - a.add("4.2.0"); + a.add("4.5.0"); } private void setProperty(JsonObject npm, String name, String value) {