diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/CanonicalType.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/CanonicalType.java index a68df031f..f9066db63 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/CanonicalType.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/model/CanonicalType.java @@ -99,6 +99,15 @@ public class CanonicalType extends UriType { } return null; } - + public boolean hasVersion() { + return getValue() != null && getValue().contains("|"); + } + + public void addVersion(String version) { + if (version != null) { + setValue(getValue()+"|"+version); + } + } + } \ No newline at end of file diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonElement.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonElement.java index c543d8674..dbd43cf61 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonElement.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonElement.java @@ -76,6 +76,7 @@ public abstract class JsonElement { return type() == JsonElementType.STRING; } + public boolean isJsonNumber() { return type() == JsonElementType.NUMBER; } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonObject.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonObject.java index 855a6e76b..fd05402a6 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonObject.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/model/JsonObject.java @@ -471,5 +471,12 @@ public class JsonObject extends JsonElement { properties.clear(); propMap.clear(); } + + public boolean isJsonString(String name) { + return has(name) && get(name).type() == JsonElementType.STRING; + } + public boolean isJsonBoolean(String name) { + return has(name) && get(name).type() == JsonElementType.BOOLEAN; + } } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java index d99d3c0a4..2a4e5702b 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java @@ -3,10 +3,12 @@ package org.hl7.fhir.validation.cli.model; import java.io.FileInputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import com.google.gson.annotations.SerializedName; @@ -221,7 +223,7 @@ public class CliContext { @JsonProperty("modeParams") @SerializedName("modeParams") private - List modeParams = new ArrayList(); + Set modeParams = new HashSet(); @JsonProperty("mode") @SerializedName("mode") @@ -779,7 +781,7 @@ public class CliContext { @SerializedName("modeParams") @JsonProperty("modeParams") - public List getModeParams() { + public Set getModeParams() { return modeParams; } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/RePackageTask.java similarity index 79% rename from org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java rename to org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/RePackageTask.java index 942d5ac09..59e3833c0 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/RePackageTask.java @@ -12,11 +12,11 @@ import org.hl7.fhir.validation.cli.model.CliContext; import org.hl7.fhir.validation.cli.services.ValidationService; import org.hl7.fhir.validation.cli.utils.Display; import org.hl7.fhir.validation.cli.utils.EngineMode; -import org.hl7.fhir.validation.special.ExpansionPackageGenerator; -import org.hl7.fhir.validation.special.ExpansionPackageGenerator.ExpansionPackageGeneratorOutputType; -import org.hl7.fhir.validation.special.ExpansionPackageGenerator.ExpansionPackageGeneratorScope; +import org.hl7.fhir.validation.special.PackageReGenerator; +import org.hl7.fhir.validation.special.PackageReGenerator.ExpansionPackageGeneratorOutputType; +import org.hl7.fhir.validation.special.PackageReGenerator.ExpansionPackageGeneratorScope; -public class TxPackTask extends ValidationEngineTask { +public class RePackageTask extends ValidationEngineTask { @Override public String getName() { @@ -35,7 +35,7 @@ public class TxPackTask extends ValidationEngineTask { @Override public boolean shouldExecuteTask(CliContext cliContext, String[] args) { - return cliContext.getMode() == EngineMode.TX_PACK; + return cliContext.getMode() == EngineMode.RE_PACKAGE; } @Override @@ -45,7 +45,6 @@ public class TxPackTask extends ValidationEngineTask { @Override public void executeTask(ValidationService validationService, ValidationEngine validationEngine, CliContext cliContext, String[] args, TimeTracker tt, TimeTracker.Session tts) throws Exception { - String pid = cliContext.getPackageName(); boolean json = cliContext.getFormat() != FhirFormat.XML; String output = cliContext.getOutput(); File f = ManagedFileAccess.file(output); @@ -66,13 +65,13 @@ public class TxPackTask extends ValidationEngineTask { } if (c < args.length - 1) { switch (args[c+1].toLowerCase()) { - case "ig" : + case "ig" : scope = ExpansionPackageGeneratorScope.IG_ONLY; break; case "igs" : scope = ExpansionPackageGeneratorScope.ALL_IGS; break; - case "core" : + case "core" : scope = ExpansionPackageGeneratorScope.EVERYTHING; break; default: @@ -80,11 +79,15 @@ public class TxPackTask extends ValidationEngineTask { } } IWorkerContext ctxt = validationEngine.getContext(); - ExpansionPackageGenerator ep = new ExpansionPackageGenerator().setContext(ctxt).setPackageId(pid).setScope(scope); + PackageReGenerator ep = new PackageReGenerator().setContext(ctxt).setScope(scope); + for (String s : cliContext.getIgs()) { + ep.addPackage(s); + } if (cliContext.getExpansionParameters() != null) { validationEngine.loadExpansionParameters(cliContext.getExpansionParameters()); } - ep.setOutput(output).setOutputType(t); + ep.setOutput(output).setOutputType(t).setJson(json); + ep.setModes(cliContext.getModeParams()); ep.generateExpansionPackage(); } } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java index acd2375c6..872e6f5fc 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java @@ -50,7 +50,10 @@ public class Params { public static final String HINT_ABOUT_NON_MUST_SUPPORT = "-hintAboutNonMustSupport"; public static final String TO_VERSION = "-to-version"; public static final String TX_PACK = "-tx-pack"; + public static final String RE_PACK = "-re-package"; public static final String PACKAGE_NAME = "-package-name"; + public static final String PIN = "-pin"; + public static final String EXPAND = "-expand"; public static final String DO_NATIVE = "-do-native"; public static final String NO_NATIVE = "-no-native"; public static final String COMPILE = "-compile"; @@ -345,8 +348,38 @@ public class Params { cliContext.setPackageName(args[++i]); cliContext.setMode(EngineMode.CODEGEN); } else if (args[i].equals(TX_PACK)) { - cliContext.setPackageName(args[++i]); - cliContext.setMode(EngineMode.TX_PACK); + cliContext.setMode(EngineMode.RE_PACKAGE); + String pn = args[++i]; + if (pn != null) { + if (pn.contains(",")) { + for (String s : pn.split("\\,")) { + cliContext.getIgs().add(s); + } + } else { + cliContext.getIgs().add(pn); + } + } + cliContext.getModeParams().add("tx"); + cliContext.getModeParams().add("expansions"); + } else if (args[i].equals(RE_PACK)) { + cliContext.setMode(EngineMode.RE_PACKAGE); + String pn = args[++i]; + if (pn != null) { + if (pn.contains(",")) { + for (String s : pn.split("\\,")) { + cliContext.getIgs().add(s); + } + } else { + cliContext.getIgs().add(pn); + } + } + cliContext.getModeParams().add("tx"); + cliContext.getModeParams().add("cnt"); + cliContext.getModeParams().add("api"); + } else if (args[i].equals(PIN)) { + cliContext.getModeParams().add("pin"); + } else if (args[i].equals(EXPAND)) { + cliContext.getModeParams().add("expand"); } else if (args[i].equals(DO_NATIVE)) { cliContext.setCanDoNative(true); } else if (args[i].equals(NO_NATIVE)) { diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java deleted file mode 100644 index e771c2e26..000000000 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java +++ /dev/null @@ -1,409 +0,0 @@ -package org.hl7.fhir.validation.special; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.hl7.fhir.convertors.txClient.TerminologyClientFactory; -import org.hl7.fhir.exceptions.FHIRException; -import org.hl7.fhir.r5.context.ContextUtilities; -import org.hl7.fhir.r5.context.IWorkerContext; -import org.hl7.fhir.r5.context.SimpleWorkerContext; -import org.hl7.fhir.r5.formats.IParser.OutputStyle; -import org.hl7.fhir.r5.formats.JsonParser; -import org.hl7.fhir.r5.formats.XmlParser; -import org.hl7.fhir.r5.model.CanonicalType; -import org.hl7.fhir.r5.model.ElementDefinition; -import org.hl7.fhir.r5.model.Parameters; -import org.hl7.fhir.r5.model.StructureDefinition; -import org.hl7.fhir.r5.model.ValueSet; -import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent; -import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome; -import org.hl7.fhir.r5.utils.NPMPackageGenerator; -import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; -import org.hl7.fhir.utilities.TextFile; -import org.hl7.fhir.utilities.Utilities; -import org.hl7.fhir.utilities.VersionUtilities; -import org.hl7.fhir.utilities.ZipGenerator; -import org.hl7.fhir.utilities.json.model.JsonObject; -import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager; -import org.hl7.fhir.utilities.npm.NpmPackage; -import org.hl7.fhir.validation.IgLoader; -import org.hl7.fhir.validation.special.ExpansionPackageGenerator.TerminologyResourceEntry; - -/** - * Given a package id, and an expansion parameters, - * get a list of all the value sets in the package, optionally with expansions. - * Parameters: - * - * - scope: this ig | all igs | all igs + core - * - expansions: true | false - * - output type: folder | zip | tgz - */ - -public class ExpansionPackageGenerator { - - public static class TerminologyResourceEntry { - public ValueSet valueSet; - public Set sources = new HashSet<>(); - public String error; - } - - public static void main(String[] args) throws Exception { - new ExpansionPackageGenerator() - .setPackageId("hl7.fhir.us.davinci-alerts") - .setJson(true) - .setOutputType(ExpansionPackageGeneratorOutputType.TGZ) - .setOutput("/Users/grahamegrieve/temp/vs-output.tgz") - .generateExpansionPackage(); - } - public enum ExpansionPackageGeneratorOutputType { - FOLDER, ZIP, TGZ - } - - public enum ExpansionPackageGeneratorScope { - IG_ONLY, ALL_IGS, EVERYTHING - } - - private String packageId; - private Parameters expansionParameters = new Parameters(); - private ExpansionPackageGeneratorScope scope = ExpansionPackageGeneratorScope.EVERYTHING; - private boolean expansions; - private String output; - private ExpansionPackageGeneratorOutputType outputType; - private boolean hierarchical; - private boolean json; - private IWorkerContext context; - - - public String getPackageId() { - return packageId; - } - - public ExpansionPackageGenerator setPackageId(String packageId) { - this.packageId = packageId; - return this; - } - - public Parameters getExpansionParameters() { - return expansionParameters; - } - - public ExpansionPackageGenerator setExpansionParameters(Parameters expansionParameters) { - this.expansionParameters = expansionParameters; - return this; - } - - public ExpansionPackageGeneratorScope getScope() { - return scope; - } - - public ExpansionPackageGenerator setScope(ExpansionPackageGeneratorScope scope) { - this.scope = scope; - return this; - } - - public boolean isExpansions() { - return expansions; - } - - public ExpansionPackageGenerator setExpansions(boolean expansions) { - this.expansions = expansions; - return this; - } - - public String getOutput() { - return output; - } - - public ExpansionPackageGenerator setOutput(String output) { - this.output = output; - return this; - } - - public ExpansionPackageGeneratorOutputType getOutputType() { - return outputType; - } - - public ExpansionPackageGenerator setOutputType(ExpansionPackageGeneratorOutputType outputType) { - this.outputType = outputType; - return this; - } - - public boolean isHierarchical() { - return hierarchical; - } - - public ExpansionPackageGenerator setHierarchical(boolean hierarchical) { - this.hierarchical = hierarchical; - return this; - } - - public boolean isJson() { - return json; - } - - public ExpansionPackageGenerator setJson(boolean json) { - this.json = json; - return this; - } - - - public IWorkerContext getContext() { - return context; - } - - public ExpansionPackageGenerator setContext(IWorkerContext context) { - this.context = context; - return this; - } - - - private ContextUtilities cu; - - private Map entries = new HashMap<>(); - - public void generateExpansionPackage() throws IOException { - if (output == null) { - throw new Error("No output"); - } - - - var npm = load(); - System.out.println("Finding ValueSets"); - for (String res : npm.listResources("StructureDefinition")) { - StructureDefinition sd = (StructureDefinition) new JsonParser().parse(npm.loadResource(res)); - processSD(sd, npm.id()); - - } - System.out.println("Generating Expansions"); - for (String n : Utilities.sorted(entries.keySet())) { - TerminologyResourceEntry e = entries.get(n); - try { - System.out.print("Generate Expansion for "+n+" ... "); - ValueSetExpansionOutcome exp = context.expandVS(e.valueSet, true, hierarchical); - if (exp.isOk()) { - e.valueSet.setExpansion(exp.getValueset().getExpansion()); - System.out.println("OK"); - } else { - e.valueSet.setExpansion(null); - e.error = exp.getError(); - System.out.println(exp.getError()); - } - } catch (Exception ex) { - System.out.println("Error= "+ex.getMessage()); - e.error = ex.getMessage(); - } - } - - System.out.println("Producing Output"); - switch (outputType) { - case FOLDER: - produceFolder(); - break; - case TGZ: - producePackage(npm); - break; - case ZIP: - produceZip(); - break; - default: - break; - - } - System.out.println("Done"); - } - - private void produceZip() throws IOException { - ZipGenerator zip = new ZipGenerator(output); - Set names = new HashSet<>(); - names.add("manifest"); - if (json) { - zip.addBytes("manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false); - } else { - zip.addBytes("manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false); - } - StringBuilder b = new StringBuilder(); - for (String n : Utilities.sorted(entries.keySet())) { - TerminologyResourceEntry e = entries.get(n); - String name = e.valueSet.getIdBase(); - int i = 0; - while (names.contains(name)) { - i++; - name = e.valueSet.getIdBase()+i; - } - names.add(name); - if (e.error == null) { - b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); - } else { - b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); - } - if (json) { - zip.addBytes(name+".json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet), false); - } else { - zip.addBytes(name+".xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet), false); - } - } - zip.addBytes("valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8), false); - zip.close(); - } - - private void producePackage(NpmPackage npm) throws FHIRException, IOException { - JsonObject j = new JsonObject(); - j.add("name", npm.id()+".custom.extensions"); - j.add("version", npm.version()); - j.add("tools-version", 3); - j.add("type", "Conformance"); - j.forceArray("fhirVersions").add(npm.fhirVersion()); - j.forceObject("dependencies").add(VersionUtilities.packageForVersion(npm.fhirVersion()), npm.fhirVersion()); - - NPMPackageGenerator gen = new NPMPackageGenerator(output, j, new Date(), true); - - Set names = new HashSet<>(); - names.add("manifest"); - if (json) { - gen.addFile("package", "manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters)); - } else { - gen.addFile("package", "manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters)); - } - StringBuilder b = new StringBuilder(); - for (String n : Utilities.sorted(entries.keySet())) { - TerminologyResourceEntry e = entries.get(n); - String name = e.valueSet.getIdBase(); - int i = 0; - while (names.contains(name)) { - i++; - name = e.valueSet.getIdBase()+i; - } - names.add(name); - if (e.error == null) { - b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); - } else { - b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); - } - if (json) { - gen.addFile("package",name+".json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet)); - } else { - gen.addFile("package",name+".xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet)); - } - } - gen.addFile("other","valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8)); - gen.finish(); - } - - private void produceFolder() throws IOException { - Utilities.createDirectory(output); - Utilities.clearDirectory(output); - Set names = new HashSet<>(); - names.add("manifest"); - if (json) { - new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.json")), expansionParameters); - } else { - new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.xml")), expansionParameters); - } - StringBuilder b = new StringBuilder(); - for (String n : Utilities.sorted(entries.keySet())) { - TerminologyResourceEntry e = entries.get(n); - String name = e.valueSet.getIdBase(); - int i = 0; - while (names.contains(name)) { - i++; - name = e.valueSet.getIdBase()+i; - } - names.add(name); - if (e.error == null) { - b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); - } else { - b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); - } - if (json) { - new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, name+".json")), e.valueSet); - } else { - new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, name+".xml")), e.valueSet); - } - } - TextFile.stringToFile(b.toString(), Utilities.path(output, "valuesets.csv")); - } - - private void processSD(StructureDefinition sd, String packageId) { -// System.out.println("Found Structure: "+sd.getVersionedUrl()); - for (ElementDefinition ed : sd.getDifferential().getElement()) { - if (ed.hasBinding() && ed.getBinding().hasValueSet()) { -// TerminologyResourceEntry e = new TerminologyResourceEntry(); - processValueSet(ed.getBinding().getValueSet(), packageId+":"+sd.getVersionedUrl()+"#"+ed.getId()); - } - } - if (scope != ExpansionPackageGeneratorScope.IG_ONLY && sd.getBaseDefinition() != null) { - StructureDefinition bsd = context.fetchResource(StructureDefinition.class, sd.getBaseDefinition()); - if (!bsd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition") || scope == ExpansionPackageGeneratorScope.EVERYTHING) { - processSD(bsd, bsd.getSourcePackage().getVID()); - } - } - } - - - private void processValueSet(String valueSet, String source) { - String url = cu.pinValueSet(valueSet); - ValueSet vs = context.fetchResource(ValueSet.class, url); - if (vs != null) { - TerminologyResourceEntry e = entries.get(vs.getVersionedUrl()); - if (e == null) { - e = new TerminologyResourceEntry(); - e.sources.add(source); - e.valueSet = vs; - entries.put(vs.getVersionedUrl(), e); - source = vs.getSourcePackage().getVID()+":"+vs.getVersionedUrl(); - for (ConceptSetComponent inc : vs.getCompose().getInclude()) { - for (CanonicalType v : inc.getValueSet()) { - if (v.hasValue()) { - processValueSet(v.primitiveValue(), source); - } - } - } - for (ConceptSetComponent inc : vs.getCompose().getExclude()) { - for (CanonicalType v : inc.getValueSet()) { - if (v.hasValue()) { - processValueSet(v.primitiveValue(), source); - } - } - } - } else { - e.sources.add(source); - } - - } else { - System.out.println("Unable to resolve value set "+valueSet); - } - - } - - private NpmPackage load() throws IOException { - FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager.Builder().build(); - NpmPackage npm = pcm.loadPackage(packageId); - if (context == null) { - String v = npm.fhirVersion(); - NpmPackage core = pcm.loadPackage(VersionUtilities.packageForVersion(v)); - NpmPackage tho = pcm.loadPackage("hl7.terminology"); - System.out.println("Load FHIR from "+core.name()+"#"+core.version()); - SimpleWorkerContext ctxt = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).fromPackage(core); - TerminologyClientFactory factory = new TerminologyClientFactory(ctxt.getVersion()); - ctxt.connectToTSServer(factory, "http://tx.fhir.org", ctxt.getUserAgent(), null, true); - var loader = new IgLoader(pcm, ctxt, ctxt.getVersion()); - loader.loadPackage(tho, true); - loader.loadPackage(npm, true); - context = ctxt; - } else { - var loader = new IgLoader(pcm, (SimpleWorkerContext) context, context.getVersion()); - loader.loadPackage(npm, true); - } - context.setExpansionParameters(expansionParameters); - cu = new ContextUtilities(context); - return npm; - } - -} diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/PackageReGenerator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/PackageReGenerator.java new file mode 100644 index 000000000..a2dc6eb51 --- /dev/null +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/PackageReGenerator.java @@ -0,0 +1,508 @@ +package org.hl7.fhir.validation.special; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.hl7.fhir.convertors.txClient.TerminologyClientFactory; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.r5.context.ContextUtilities; +import org.hl7.fhir.r5.context.IWorkerContext; +import org.hl7.fhir.r5.context.SimpleWorkerContext; +import org.hl7.fhir.r5.formats.IParser.OutputStyle; +import org.hl7.fhir.r5.formats.JsonParser; +import org.hl7.fhir.r5.formats.XmlParser; +import org.hl7.fhir.r5.model.Base; +import org.hl7.fhir.r5.model.CanonicalResource; +import org.hl7.fhir.r5.model.CanonicalType; +import org.hl7.fhir.r5.model.CapabilityStatement; +import org.hl7.fhir.r5.model.CodeSystem; +import org.hl7.fhir.r5.model.ElementDefinition; +import org.hl7.fhir.r5.model.ElementDefinition.TypeRefComponent; +import org.hl7.fhir.r5.model.NamingSystem; +import org.hl7.fhir.r5.model.OperationDefinition; +import org.hl7.fhir.r5.model.Parameters; +import org.hl7.fhir.r5.model.Property; +import org.hl7.fhir.r5.model.Resource; +import org.hl7.fhir.r5.model.SearchParameter; +import org.hl7.fhir.r5.model.StructureDefinition; +import org.hl7.fhir.r5.model.ValueSet; +import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome; +import org.hl7.fhir.r5.utils.NPMPackageGenerator; +import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.VersionUtilities; +import org.hl7.fhir.utilities.ZipGenerator; +import org.hl7.fhir.utilities.json.model.JsonObject; +import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager; +import org.hl7.fhir.utilities.npm.NpmPackage; +import org.hl7.fhir.validation.IgLoader; +import org.hl7.fhir.validation.special.PackageReGenerator.TerminologyResourceEntry; + +/** + * Given a package id, and an expansion parameters, + * get a list of all the value sets in the package, optionally with expansions. + * Parameters: + * + * - scope: this ig | all igs | all igs + core + * - expansions: true | false + * - output type: folder | zip | tgz + */ + +public class PackageReGenerator { + + public static class TerminologyResourceEntry { + public ValueSet valueSet; + public Set sources = new HashSet<>(); + public String error; + } + + public static void main(String[] args) throws Exception { + new PackageReGenerator() + .addPackage("hl7.fhir.us.davinci-alerts") + .setJson(true) + .setOutputType(ExpansionPackageGeneratorOutputType.TGZ) + .setOutput("/Users/grahamegrieve/temp/vs-output.tgz") + .generateExpansionPackage(); + } + + public enum ExpansionPackageGeneratorOutputType { + FOLDER, ZIP, TGZ + } + + public enum ExpansionPackageGeneratorScope { + IG_ONLY, ALL_IGS, EVERYTHING + } + + private List packages = new ArrayList(); + private Parameters expansionParameters = new Parameters(); + private ExpansionPackageGeneratorScope scope = ExpansionPackageGeneratorScope.EVERYTHING; + private String output; + private ExpansionPackageGeneratorOutputType outputType; + private boolean hierarchical; + private boolean json; + private IWorkerContext context; + + + public PackageReGenerator addPackage(String packageId) { + packages.add(packageId); + return this; + } + + public Parameters getExpansionParameters() { + return expansionParameters; + } + + public PackageReGenerator setExpansionParameters(Parameters expansionParameters) { + this.expansionParameters = expansionParameters; + return this; + } + + public ExpansionPackageGeneratorScope getScope() { + return scope; + } + + public PackageReGenerator setScope(ExpansionPackageGeneratorScope scope) { + this.scope = scope; + return this; + } + + public String getOutput() { + return output; + } + + public PackageReGenerator setOutput(String output) { + this.output = output; + return this; + } + + public ExpansionPackageGeneratorOutputType getOutputType() { + return outputType; + } + + public PackageReGenerator setOutputType(ExpansionPackageGeneratorOutputType outputType) { + this.outputType = outputType; + return this; + } + + public boolean isHierarchical() { + return hierarchical; + } + + public PackageReGenerator setHierarchical(boolean hierarchical) { + this.hierarchical = hierarchical; + return this; + } + + public boolean isJson() { + return json; + } + + public PackageReGenerator setJson(boolean json) { + this.json = json; + return this; + } + + + public IWorkerContext getContext() { + return context; + } + + public PackageReGenerator setContext(IWorkerContext context) { + this.context = context; + return this; + } + + + private ContextUtilities cu; + + private Map entries = new HashMap<>(); + private Set modeParams; + private List resources = new ArrayList(); + + public void generateExpansionPackage() throws IOException { + if (output == null) { + throw new Error("No output"); + } + + var list = load(); + for (NpmPackage npm : list) { + if (modeParams.contains("api")) { + System.out.println("Processing CapabilityStatements"); + for (String res : npm.listResources("CapabilityStatement")) { + CapabilityStatement cs = (CapabilityStatement) new JsonParser().parse(npm.loadResource(res)); + processResource(cs); + } + System.out.println("Processing OperationDefinitions"); + for (String res : npm.listResources("OperationDefinition")) { + OperationDefinition op = (OperationDefinition) new JsonParser().parse(npm.loadResource(res)); + processResource(op); + } + System.out.println("Processing SearchParameters"); + for (String res : npm.listResources("SearchParameter")) { + SearchParameter sp = (SearchParameter) new JsonParser().parse(npm.loadResource(res)); + processResource(sp); + } + } + if (modeParams.contains("tx")) { + System.out.println("Processing CodeSystems"); + for (String res : npm.listResources("CodeSystem")) { + CodeSystem cs = (CodeSystem) new JsonParser().parse(npm.loadResource(res)); + processResource(cs); + } + System.out.println("Processing ValueSets"); + for (String res : npm.listResources("ValueSet")) { + ValueSet vs = (ValueSet) new JsonParser().parse(npm.loadResource(res)); + processResource(vs); + } + System.out.println("Processing NamingSystems"); + for (String res : npm.listResources("NamingSystem")) { + NamingSystem ns = (NamingSystem) new JsonParser().parse(npm.loadResource(res)); + processResource(ns); + } + } + if (modeParams.contains("cnt")) { + System.out.println("Processing StructureDefinitions"); + for (String res : npm.listResources("StructureDefinition")) { + StructureDefinition cs = (StructureDefinition) new JsonParser().parse(npm.loadResource(res)); + processResource(cs); + } + } + System.out.println("Processing Bindings"); + for (String res : npm.listResources("StructureDefinition")) { + StructureDefinition sd = (StructureDefinition) new JsonParser().parse(npm.loadResource(res)); + processSD(sd, npm.id()); + } + if (modeParams.contains("expansions")) { + System.out.println("Generating Expansions"); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + try { + System.out.print("Generate Expansion for "+n+" ... "); + ValueSetExpansionOutcome exp = context.expandVS(e.valueSet, true, hierarchical); + if (exp.isOk()) { + e.valueSet.setExpansion(exp.getValueset().getExpansion()); + System.out.println("OK"); + } else { + e.valueSet.setExpansion(null); + e.error = exp.getError(); + System.out.println(exp.getError()); + } + } catch (Exception ex) { + System.out.println("Error= "+ex.getMessage()); + e.error = ex.getMessage(); + } + } + } + } + + switch (outputType) { + case FOLDER: + produceFolder(); + break; + case TGZ: + producePackage(); + break; + case ZIP: + produceZip(); + break; + default: + break; + + } + System.out.println("Done"); + } + + private void processResource(CanonicalResource res) { + resources.add(res); + if (modeParams.contains("pin")) { + processBase(res, res); + } + } + + + private void processBase(CanonicalResource src, Base b) { + for (Property p : b.children()) { + for (Base v : p.getValues()) { + processBase(src, v); + } + } + if (b instanceof CanonicalType) { + CanonicalType ct = (CanonicalType) b; + if (!ct.hasVersion()) { + Resource res = context.fetchResource(Resource.class, ct.getValue(), src); + if (res != null && res instanceof CanonicalResource) { + CanonicalResource cr = (CanonicalResource) res; + ct.addVersion(cr.getVersion()); + } + } + } + } + + private void produceZip() throws IOException { + System.out.println("Producing Output in Zip "+output); + ZipGenerator zip = new ZipGenerator(output); + Set names = new HashSet<>(); + names.add("manifest"); + if (json) { + zip.addBytes("manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false); + } else { + zip.addBytes("manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false); + } + if (modeParams.contains("expansions")) { + StringBuilder b = new StringBuilder(); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + String name = e.valueSet.getIdBase(); + int i = 0; + while (names.contains(name)) { + i++; + name = e.valueSet.getIdBase()+i; + } + names.add(name); + if (e.error == null) { + b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } else { + b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } + zip.addBytes(name+".json", composeResource(e.valueSet), false); + } + zip.addBytes("valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8), false); + } + zip.close(); + } + + private void producePackage() throws FHIRException, IOException { + System.out.println("Producing Output Package in "+output); + JsonObject j = new JsonObject(); + j.add("name", "custom.generated"); + j.add("version", "0.0.1"); + j.add("tools-version", 3); + j.add("type", "Conformance"); + j.forceArray("fhirVersions").add(context.getVersion()); + j.forceObject("dependencies").add(VersionUtilities.packageForVersion(context.getVersion()), context.getVersion()); + + NPMPackageGenerator gen = new NPMPackageGenerator(output, j, new Date(), true); + + Set names = new HashSet<>(); + names.add("manifest"); + if (json) { + gen.addFile("package", "manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters)); + } else { + gen.addFile("package", "manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters)); + } + if (modeParams.contains("expansions")) { + StringBuilder b = new StringBuilder(); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + String name = e.valueSet.getIdBase(); + int i = 0; + while (names.contains(name)) { + i++; + name = e.valueSet.getIdBase()+i; + } + names.add(name); + if (e.error == null) { + b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } else { + b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } + gen.addFile("package",name+".json", composeResource(e.valueSet)); + } + gen.addFile("other","valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8)); + } + gen.finish(); + } + + private void produceFolder() throws IOException { + System.out.println("Producing Output in folder "+output); + Utilities.createDirectory(output); + Utilities.clearDirectory(output); + Set names = new HashSet<>(); + names.add("manifest"); + if (json) { + new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.json")), expansionParameters); + } else { + new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.xml")), expansionParameters); + } + for (CanonicalResource cr : resources) { + TextFile.bytesToFile(composeResource(cr), Utilities.path(output, cr.fhirType()+"-"+cr.getIdBase()+(json? ".json" : ".xml"))); + } + if (modeParams.contains("expansions")) { + StringBuilder b = new StringBuilder(); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + String name = "ValueSet-"+e.valueSet.getIdBase()+"-expansion"; + int i = 0; + while (names.contains(name)) { + i++; + name = e.valueSet.getIdBase()+i; + } + names.add(name); + if (e.error == null) { + b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } else { + b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } + TextFile.bytesToFile(composeResource(e.valueSet), Utilities.path(output, name+(json? ".json" : ".xml"))); + } + TextFile.stringToFile(b.toString(), Utilities.path(output, "expansions.csv")); + } + } + + + private byte[] composeResource(CanonicalResource cr) throws IOException { + if (json) { + return new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(cr); + } else { + return new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(cr); + } + } + + private void processSD(StructureDefinition sd, String packageId) { + for (ElementDefinition ed : sd.getDifferential().getElement()) { + if (ed.hasBinding() && ed.getBinding().hasValueSet()) { + processValueSet(ed.getBinding().getValueSet(), packageId+":"+sd.getVersionedUrl()+"#"+ed.getId()); + } + } + if (scope != ExpansionPackageGeneratorScope.IG_ONLY && sd.getBaseDefinition() != null) { + StructureDefinition bsd = context.fetchResource(StructureDefinition.class, sd.getBaseDefinition(), sd); + if (bsd != null) { + if (!bsd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition") || scope == ExpansionPackageGeneratorScope.EVERYTHING) { + processSD(bsd, bsd.getSourcePackage().getVID()); + } + } + } + } + + + private void processValueSet(String valueSet, String source) { + String url = cu.pinValueSet(valueSet); + ValueSet vs = context.fetchResource(ValueSet.class, url); + if (vs != null) { + TerminologyResourceEntry e = entries.get(vs.getVersionedUrl()); + if (e == null) { + e = new TerminologyResourceEntry(); + e.sources.add(source); + e.valueSet = vs; + entries.put(vs.getVersionedUrl(), e); + source = vs.getSourcePackage().getVID()+":"+vs.getVersionedUrl(); + for (ConceptSetComponent inc : vs.getCompose().getInclude()) { + for (CanonicalType v : inc.getValueSet()) { + if (v.hasValue()) { + processValueSet(v.primitiveValue(), source); + } + } + } + for (ConceptSetComponent inc : vs.getCompose().getExclude()) { + for (CanonicalType v : inc.getValueSet()) { + if (v.hasValue()) { + processValueSet(v.primitiveValue(), source); + } + } + } + } else { + e.sources.add(source); + } + + } else { + System.out.println("Unable to resolve value set "+valueSet); + } + + } + + private List load() throws IOException { + List list = new ArrayList(); + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager.Builder().build(); + for (String packageId : packages) { + NpmPackage npm = pcm.loadPackage(packageId); + if (context == null) { + String v = npm.fhirVersion(); + NpmPackage core = pcm.loadPackage(VersionUtilities.packageForVersion(v)); + NpmPackage tho = pcm.loadPackage("hl7.terminology"); + System.out.println("Load FHIR from "+core.name()+"#"+core.version()); + SimpleWorkerContext ctxt = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).fromPackage(core); + TerminologyClientFactory factory = new TerminologyClientFactory(ctxt.getVersion()); + ctxt.connectToTSServer(factory, "http://tx.fhir.org", ctxt.getUserAgent(), null, true); + var loader = new IgLoader(pcm, ctxt, ctxt.getVersion()); + loader.loadPackage(tho, true); + loader.loadPackage(npm, true); + context = ctxt; + context.setExpansionParameters(expansionParameters); + if (scope == ExpansionPackageGeneratorScope.EVERYTHING) { +// list.add(core); +// list.add(tho); + } + } else { + var loader = new IgLoader(pcm, (SimpleWorkerContext) context, context.getVersion()); + loader.loadPackage(npm, true); + if (scope == ExpansionPackageGeneratorScope.ALL_IGS) { + for (NpmPackage p : loader.getPackageList()) { + // nothing? + } + } + } + list.add(npm); + } + if (cu == null) { + cu = new ContextUtilities(context); + } + return list; + } + + public void setModes(Set modeParams) { + this.modeParams = modeParams; + + } + +} diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java index 3c5d3c4b9..385547b83 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java @@ -93,7 +93,7 @@ public class ValidatorCliTests { CodeGenTask codeGenTask; @Spy - TxPackTask txPackTask; + RePackageTask txPackTask; @Spy InstanceFactoryTask instanceFactoryTask;