From 5b73c4762de10e2bee60a69c2169e7d7dadab4a6 Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sun, 8 Dec 2024 18:08:29 +0300 Subject: [PATCH] Add support for terminology extraction and support for expansion parameters when validating --- .../org/hl7/fhir/r5/elementmodel/Manager.java | 8 + .../hl7/fhir/validation/ValidationEngine.java | 21 + .../org/hl7/fhir/validation/ValidatorCli.java | 1 + .../fhir/validation/cli/model/CliContext.java | 45 +- .../fhir/validation/cli/tasks/TxPackTask.java | 83 ++++ .../validation/cli/tasks/ValidateTask.java | 4 + .../fhir/validation/cli/utils/EngineMode.java | 3 +- .../hl7/fhir/validation/cli/utils/Params.java | 11 + .../special/ExpansionPackageGenerator.java | 409 ++++++++++++++++++ .../special/TxServiceTestHelper.java | 6 + .../src/main/resources/help/tx-pack.txt | 11 + .../fhir/validation/ValidatorCliTests.java | 4 + 12 files changed, 604 insertions(+), 2 deletions(-) create mode 100644 org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java create mode 100644 org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java create mode 100644 org.hl7.fhir.validation/src/main/resources/help/tx-pack.txt diff --git a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Manager.java b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Manager.java index d17c8b112..c656243df 100644 --- a/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Manager.java +++ b/org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Manager.java @@ -111,6 +111,14 @@ public class Manager { } return null; } + + public static FhirFormat fromCode(String code) { + FhirFormat fmt = getFhirFormat(code); + if (fmt == null) { + fmt = readFromMimeType(code); + } + return fmt; + } } public static List parse(IWorkerContext context, InputStream source, FhirFormat inputFormat) throws FHIRFormatError, DefinitionException, IOException, FHIRException { diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java index 11e8022bc..f89c9a7ab 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java @@ -2,6 +2,7 @@ package org.hl7.fhir.validation; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -1295,4 +1296,24 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP return ReferenceValidationPolicy.IGNORE; } + public void loadExpansionParameters(String expansionParameters) { + System.out.println("Load Expansion Parameters: "+expansionParameters); + Parameters p = null; + try { + p = (Parameters) new XmlParser().parse(new FileInputStream(expansionParameters)); + } catch (Exception e) { + } + if (p == null) { + try { + p = (Parameters) new JsonParser().parse(new FileInputStream(expansionParameters)); + } catch (Exception e) { + System.out.println("Unable to load expansion parameters '"+expansionParameters+"' as either xml or json: "+e.getMessage()); + throw new FHIRException("Unable to load expansion parameters '"+expansionParameters+"' as either xml or json: "+e.getMessage()); + } + } + context.setExpansionParameters(p); + + + } + } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java index 3b38aaf36..c4c699f11 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java @@ -137,6 +137,7 @@ public class ValidatorCli { new TransformTask(), new VersionTask(), new CodeGenTask(), + new TxPackTask(), defaultCliTask); } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java index 3cabac33c..7c2338361 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java @@ -1,5 +1,6 @@ package org.hl7.fhir.validation.cli.model; +import java.io.FileInputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -8,6 +9,11 @@ import java.util.Map; import java.util.Objects; import com.google.gson.annotations.SerializedName; + +import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; +import org.hl7.fhir.r5.formats.JsonParser; +import org.hl7.fhir.r5.formats.XmlParser; +import org.hl7.fhir.r5.model.Parameters; import org.hl7.fhir.r5.terminologies.JurisdictionUtilities; import org.hl7.fhir.r5.utils.validation.BundleValidationRule; import org.hl7.fhir.r5.utils.validation.constants.BestPracticeWarningLevel; @@ -322,6 +328,16 @@ public class CliContext { private String advisorFile; + @JsonProperty("expansionParameters") + @SerializedName("expansionParameters") + private + String expansionParameters; + + @JsonProperty("format") + @SerializedName("format") + private + FhirFormat format; + @SerializedName("baseEngine") @JsonProperty("baseEngine") public String getBaseEngine() { @@ -1146,6 +1162,8 @@ public class CliContext { Objects.equals(unknownCodeSystemsCauseErrors, that.unknownCodeSystemsCauseErrors) && Objects.equals(noExperimentalContent, that.noExperimentalContent) && Objects.equals(advisorFile, that.advisorFile) && + Objects.equals(expansionParameters, that.expansionParameters) && + Objects.equals(format, that.format) && Objects.equals(watchSettleTime, that.watchSettleTime); } @@ -1154,7 +1172,7 @@ public class CliContext { return Objects.hash(baseEngine, doNative, extensions, hintAboutNonMustSupport, recursive, doDebug, assumeValidRestReferences, canDoNative, noInternalCaching, noExtensibleBindingMessages, noInvariants, displayWarnings, wantInvariantsInMessages, map, output, outputSuffix, htmlOutput, txServer, sv, txLog, txCache, mapLog, lang, srcLang, tgtLang, fhirpath, snomedCT, targetVer, packageName, igs, questionnaireMode, level, profiles, options, sources, inputs, mode, locale, locations, crumbTrails, showMessageIds, forPublication, showTimes, allowExampleUrls, outputStyle, jurisdiction, noUnicodeBiDiControlChars, - watchMode, watchScanDelay, watchSettleTime, bestPracticeLevel, unknownCodeSystemsCauseErrors, noExperimentalContent, advisorFile, htmlInMarkdownCheck, allowDoubleQuotesInFHIRPath, checkIPSCodes); + watchMode, watchScanDelay, watchSettleTime, bestPracticeLevel, unknownCodeSystemsCauseErrors, noExperimentalContent, advisorFile, expansionParameters, format, htmlInMarkdownCheck, allowDoubleQuotesInFHIRPath, checkIPSCodes); } @Override @@ -1219,6 +1237,8 @@ public class CliContext { ", unknownCodeSystemsCauseErrors=" + unknownCodeSystemsCauseErrors + ", noExperimentalContent=" + noExperimentalContent + ", advisorFile=" + advisorFile + + ", expansionParameters=" + expansionParameters + + ", format=" + format + '}'; } @@ -1325,5 +1345,28 @@ public class CliContext { this.advisorFile = advisorFile; } + @SerializedName("expansionParameters") + @JsonProperty("expansionParameters") + public String getExpansionParameters() { + return expansionParameters; + } + + @SerializedName("expansionParameters") + @JsonProperty("expansionParameters") + public void setExpansionParameters(String expansionParameters) { + this.expansionParameters = expansionParameters; + } + + @SerializedName("format") + @JsonProperty("format") + public FhirFormat getFormat() { + return format; + } + + @SerializedName("format") + @JsonProperty("format") + public void setFormat(FhirFormat format) { + this.format = format; + } } \ No newline at end of file diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java new file mode 100644 index 000000000..c79439399 --- /dev/null +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java @@ -0,0 +1,83 @@ +package org.hl7.fhir.validation.cli.tasks; + +import java.io.File; +import java.io.PrintStream; + +import org.hl7.fhir.r5.context.IWorkerContext; +import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; +import org.hl7.fhir.utilities.TimeTracker; +import org.hl7.fhir.validation.ValidationEngine; +import org.hl7.fhir.validation.cli.model.CliContext; +import org.hl7.fhir.validation.cli.services.ValidationService; +import org.hl7.fhir.validation.cli.utils.Display; +import org.hl7.fhir.validation.cli.utils.EngineMode; +import org.hl7.fhir.validation.special.ExpansionPackageGenerator; +import org.hl7.fhir.validation.special.ExpansionPackageGenerator.ExpansionPackageGeneratorOutputType; +import org.hl7.fhir.validation.special.ExpansionPackageGenerator.ExpansionPackageGeneratorScope; + +public class TxPackTask extends ValidationEngineTask { + + @Override + public String getName() { + return "tx-pack"; + } + + @Override + public String getDisplayName() { + return "Generate a terminology pack"; + } + + @Override + public boolean isHidden() { + return false; + } + + @Override + public boolean shouldExecuteTask(CliContext cliContext, String[] args) { + return cliContext.getMode() == EngineMode.TX_PACK; + } + + @Override + public void printHelp(PrintStream out) { + Display.displayHelpDetails(out,"help/tx-pack.txt"); + } + + @Override + public void executeTask(ValidationService validationService, ValidationEngine validationEngine, CliContext cliContext, String[] args, TimeTracker tt, TimeTracker.Session tts) throws Exception { + String pid = cliContext.getPackageName(); + boolean json = cliContext.getFormat() != FhirFormat.XML; + String output = cliContext.getOutput(); + File f = new File(output); + ExpansionPackageGeneratorOutputType t = ExpansionPackageGeneratorOutputType.FOLDER; + if (f.exists() && f.isDirectory()) { + t = ExpansionPackageGeneratorOutputType.FOLDER; + } else if (output.endsWith(".zip")) { + t = ExpansionPackageGeneratorOutputType.ZIP; + } else if (output.endsWith(".tgz")) { + t = ExpansionPackageGeneratorOutputType.TGZ; + } + ExpansionPackageGeneratorScope scope = ExpansionPackageGeneratorScope.IG_ONLY; + int c = -1; + for (int i = 0; i < args.length; i++) { + if ("-scope".equals(args[i])) { + c = i; + } + } + if (c < args.length - 1) { + switch (args[c+1].toLowerCase()) { + case "ig" : scope = ExpansionPackageGeneratorScope.IG_ONLY; + case "igs" : scope = ExpansionPackageGeneratorScope.ALL_IGS; + case "core" : scope = ExpansionPackageGeneratorScope.EVERYTHING; + default: + System.out.println("Unknown scope "+args[c+1]); + } + } + IWorkerContext ctxt = validationEngine.getContext(); + ExpansionPackageGenerator ep = new ExpansionPackageGenerator().setContext(ctxt).setPackageId(pid).setScope(scope); + if (cliContext.getExpansionParameters() != null) { + validationEngine.loadExpansionParameters(cliContext.getExpansionParameters()); + } + ep.setOutput(output).setOutputType(t); + ep.generateExpansionPackage(); + } +} diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/ValidateTask.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/ValidateTask.java index ebbdc5877..89aae7770 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/ValidateTask.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/ValidateTask.java @@ -51,6 +51,10 @@ public class ValidateTask extends ValidationEngineTask { @Override public void executeTask(ValidationService validationService, ValidationEngine validationEngine, CliContext cliContext, String[] args, TimeTracker tt, TimeTracker.Session tts) throws Exception { + if (cliContext.getExpansionParameters() != null) { + validationEngine.loadExpansionParameters(cliContext.getExpansionParameters()); + } + for (String s : cliContext.getProfiles()) { if (!validationEngine.getContext().hasResource(StructureDefinition.class, s) && !validationEngine.getContext().hasResource(ImplementationGuide.class, s)) { System.out.println(" Fetch Profile from " + s); diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/EngineMode.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/EngineMode.java index 21fcfc123..4482a7f4b 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/EngineMode.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/EngineMode.java @@ -14,5 +14,6 @@ public enum EngineMode { VERSION, RUN_TESTS, INSTALL, - CODEGEN + CODEGEN, + TX_PACK } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java index d9193e578..c6fc83bb7 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java @@ -6,6 +6,7 @@ import java.util.Arrays; import java.util.Locale; import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; import org.hl7.fhir.r5.terminologies.JurisdictionUtilities; import org.hl7.fhir.r5.utils.validation.BundleValidationRule; import org.hl7.fhir.r5.utils.validation.constants.BestPracticeWarningLevel; @@ -45,13 +46,16 @@ public class Params { public static final String EXTENSION = "-extension"; public static final String HINT_ABOUT_NON_MUST_SUPPORT = "-hintAboutNonMustSupport"; public static final String TO_VERSION = "-to-version"; + public static final String TX_PACK = "-tx-pack"; public static final String PACKAGE_NAME = "-package-name"; public static final String DO_NATIVE = "-do-native"; public static final String NO_NATIVE = "-no-native"; public static final String COMPILE = "-compile"; public static final String CODEGEN = "-codegen"; public static final String TRANSFORM = "-transform"; + public static final String FORMAT = "-format"; public static final String LANG_TRANSFORM = "-lang-transform"; + public static final String EXP_PARAMS = "-expansion-parameters"; public static final String NARRATIVE = "-narrative"; public static final String SNAPSHOT = "-snapshot"; public static final String INSTALL = "-install"; @@ -330,6 +334,9 @@ public class Params { } else if (args[i].equals(PACKAGE_NAME)) { cliContext.setPackageName(args[++i]); cliContext.setMode(EngineMode.CODEGEN); + } else if (args[i].equals(TX_PACK)) { + cliContext.setPackageName(args[++i]); + cliContext.setMode(EngineMode.TX_PACK); } else if (args[i].equals(DO_NATIVE)) { cliContext.setCanDoNative(true); } else if (args[i].equals(NO_NATIVE)) { @@ -337,9 +344,13 @@ public class Params { } else if (args[i].equals(TRANSFORM)) { cliContext.setMap(args[++i]); cliContext.setMode(EngineMode.TRANSFORM); + } else if (args[i].equals(FORMAT)) { + cliContext.setFormat(FhirFormat.fromCode(args[++i])); } else if (args[i].equals(LANG_TRANSFORM)) { cliContext.setLangTransform(args[++i]); cliContext.setMode(EngineMode.LANG_TRANSFORM); + } else if (args[i].equals(EXP_PARAMS)) { + cliContext.setExpansionParameters(args[++i]); } else if (args[i].equals(COMPILE)) { cliContext.setMap(args[++i]); cliContext.setMode(EngineMode.COMPILE); diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java new file mode 100644 index 000000000..e771c2e26 --- /dev/null +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java @@ -0,0 +1,409 @@ +package org.hl7.fhir.validation.special; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.hl7.fhir.convertors.txClient.TerminologyClientFactory; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.r5.context.ContextUtilities; +import org.hl7.fhir.r5.context.IWorkerContext; +import org.hl7.fhir.r5.context.SimpleWorkerContext; +import org.hl7.fhir.r5.formats.IParser.OutputStyle; +import org.hl7.fhir.r5.formats.JsonParser; +import org.hl7.fhir.r5.formats.XmlParser; +import org.hl7.fhir.r5.model.CanonicalType; +import org.hl7.fhir.r5.model.ElementDefinition; +import org.hl7.fhir.r5.model.Parameters; +import org.hl7.fhir.r5.model.StructureDefinition; +import org.hl7.fhir.r5.model.ValueSet; +import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent; +import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome; +import org.hl7.fhir.r5.utils.NPMPackageGenerator; +import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.VersionUtilities; +import org.hl7.fhir.utilities.ZipGenerator; +import org.hl7.fhir.utilities.json.model.JsonObject; +import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager; +import org.hl7.fhir.utilities.npm.NpmPackage; +import org.hl7.fhir.validation.IgLoader; +import org.hl7.fhir.validation.special.ExpansionPackageGenerator.TerminologyResourceEntry; + +/** + * Given a package id, and an expansion parameters, + * get a list of all the value sets in the package, optionally with expansions. + * Parameters: + * + * - scope: this ig | all igs | all igs + core + * - expansions: true | false + * - output type: folder | zip | tgz + */ + +public class ExpansionPackageGenerator { + + public static class TerminologyResourceEntry { + public ValueSet valueSet; + public Set sources = new HashSet<>(); + public String error; + } + + public static void main(String[] args) throws Exception { + new ExpansionPackageGenerator() + .setPackageId("hl7.fhir.us.davinci-alerts") + .setJson(true) + .setOutputType(ExpansionPackageGeneratorOutputType.TGZ) + .setOutput("/Users/grahamegrieve/temp/vs-output.tgz") + .generateExpansionPackage(); + } + public enum ExpansionPackageGeneratorOutputType { + FOLDER, ZIP, TGZ + } + + public enum ExpansionPackageGeneratorScope { + IG_ONLY, ALL_IGS, EVERYTHING + } + + private String packageId; + private Parameters expansionParameters = new Parameters(); + private ExpansionPackageGeneratorScope scope = ExpansionPackageGeneratorScope.EVERYTHING; + private boolean expansions; + private String output; + private ExpansionPackageGeneratorOutputType outputType; + private boolean hierarchical; + private boolean json; + private IWorkerContext context; + + + public String getPackageId() { + return packageId; + } + + public ExpansionPackageGenerator setPackageId(String packageId) { + this.packageId = packageId; + return this; + } + + public Parameters getExpansionParameters() { + return expansionParameters; + } + + public ExpansionPackageGenerator setExpansionParameters(Parameters expansionParameters) { + this.expansionParameters = expansionParameters; + return this; + } + + public ExpansionPackageGeneratorScope getScope() { + return scope; + } + + public ExpansionPackageGenerator setScope(ExpansionPackageGeneratorScope scope) { + this.scope = scope; + return this; + } + + public boolean isExpansions() { + return expansions; + } + + public ExpansionPackageGenerator setExpansions(boolean expansions) { + this.expansions = expansions; + return this; + } + + public String getOutput() { + return output; + } + + public ExpansionPackageGenerator setOutput(String output) { + this.output = output; + return this; + } + + public ExpansionPackageGeneratorOutputType getOutputType() { + return outputType; + } + + public ExpansionPackageGenerator setOutputType(ExpansionPackageGeneratorOutputType outputType) { + this.outputType = outputType; + return this; + } + + public boolean isHierarchical() { + return hierarchical; + } + + public ExpansionPackageGenerator setHierarchical(boolean hierarchical) { + this.hierarchical = hierarchical; + return this; + } + + public boolean isJson() { + return json; + } + + public ExpansionPackageGenerator setJson(boolean json) { + this.json = json; + return this; + } + + + public IWorkerContext getContext() { + return context; + } + + public ExpansionPackageGenerator setContext(IWorkerContext context) { + this.context = context; + return this; + } + + + private ContextUtilities cu; + + private Map entries = new HashMap<>(); + + public void generateExpansionPackage() throws IOException { + if (output == null) { + throw new Error("No output"); + } + + + var npm = load(); + System.out.println("Finding ValueSets"); + for (String res : npm.listResources("StructureDefinition")) { + StructureDefinition sd = (StructureDefinition) new JsonParser().parse(npm.loadResource(res)); + processSD(sd, npm.id()); + + } + System.out.println("Generating Expansions"); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + try { + System.out.print("Generate Expansion for "+n+" ... "); + ValueSetExpansionOutcome exp = context.expandVS(e.valueSet, true, hierarchical); + if (exp.isOk()) { + e.valueSet.setExpansion(exp.getValueset().getExpansion()); + System.out.println("OK"); + } else { + e.valueSet.setExpansion(null); + e.error = exp.getError(); + System.out.println(exp.getError()); + } + } catch (Exception ex) { + System.out.println("Error= "+ex.getMessage()); + e.error = ex.getMessage(); + } + } + + System.out.println("Producing Output"); + switch (outputType) { + case FOLDER: + produceFolder(); + break; + case TGZ: + producePackage(npm); + break; + case ZIP: + produceZip(); + break; + default: + break; + + } + System.out.println("Done"); + } + + private void produceZip() throws IOException { + ZipGenerator zip = new ZipGenerator(output); + Set names = new HashSet<>(); + names.add("manifest"); + if (json) { + zip.addBytes("manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false); + } else { + zip.addBytes("manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false); + } + StringBuilder b = new StringBuilder(); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + String name = e.valueSet.getIdBase(); + int i = 0; + while (names.contains(name)) { + i++; + name = e.valueSet.getIdBase()+i; + } + names.add(name); + if (e.error == null) { + b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } else { + b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } + if (json) { + zip.addBytes(name+".json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet), false); + } else { + zip.addBytes(name+".xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet), false); + } + } + zip.addBytes("valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8), false); + zip.close(); + } + + private void producePackage(NpmPackage npm) throws FHIRException, IOException { + JsonObject j = new JsonObject(); + j.add("name", npm.id()+".custom.extensions"); + j.add("version", npm.version()); + j.add("tools-version", 3); + j.add("type", "Conformance"); + j.forceArray("fhirVersions").add(npm.fhirVersion()); + j.forceObject("dependencies").add(VersionUtilities.packageForVersion(npm.fhirVersion()), npm.fhirVersion()); + + NPMPackageGenerator gen = new NPMPackageGenerator(output, j, new Date(), true); + + Set names = new HashSet<>(); + names.add("manifest"); + if (json) { + gen.addFile("package", "manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters)); + } else { + gen.addFile("package", "manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters)); + } + StringBuilder b = new StringBuilder(); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + String name = e.valueSet.getIdBase(); + int i = 0; + while (names.contains(name)) { + i++; + name = e.valueSet.getIdBase()+i; + } + names.add(name); + if (e.error == null) { + b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } else { + b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } + if (json) { + gen.addFile("package",name+".json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet)); + } else { + gen.addFile("package",name+".xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet)); + } + } + gen.addFile("other","valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8)); + gen.finish(); + } + + private void produceFolder() throws IOException { + Utilities.createDirectory(output); + Utilities.clearDirectory(output); + Set names = new HashSet<>(); + names.add("manifest"); + if (json) { + new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.json")), expansionParameters); + } else { + new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.xml")), expansionParameters); + } + StringBuilder b = new StringBuilder(); + for (String n : Utilities.sorted(entries.keySet())) { + TerminologyResourceEntry e = entries.get(n); + String name = e.valueSet.getIdBase(); + int i = 0; + while (names.contains(name)) { + i++; + name = e.valueSet.getIdBase()+i; + } + names.add(name); + if (e.error == null) { + b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } else { + b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n"); + } + if (json) { + new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, name+".json")), e.valueSet); + } else { + new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, name+".xml")), e.valueSet); + } + } + TextFile.stringToFile(b.toString(), Utilities.path(output, "valuesets.csv")); + } + + private void processSD(StructureDefinition sd, String packageId) { +// System.out.println("Found Structure: "+sd.getVersionedUrl()); + for (ElementDefinition ed : sd.getDifferential().getElement()) { + if (ed.hasBinding() && ed.getBinding().hasValueSet()) { +// TerminologyResourceEntry e = new TerminologyResourceEntry(); + processValueSet(ed.getBinding().getValueSet(), packageId+":"+sd.getVersionedUrl()+"#"+ed.getId()); + } + } + if (scope != ExpansionPackageGeneratorScope.IG_ONLY && sd.getBaseDefinition() != null) { + StructureDefinition bsd = context.fetchResource(StructureDefinition.class, sd.getBaseDefinition()); + if (!bsd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition") || scope == ExpansionPackageGeneratorScope.EVERYTHING) { + processSD(bsd, bsd.getSourcePackage().getVID()); + } + } + } + + + private void processValueSet(String valueSet, String source) { + String url = cu.pinValueSet(valueSet); + ValueSet vs = context.fetchResource(ValueSet.class, url); + if (vs != null) { + TerminologyResourceEntry e = entries.get(vs.getVersionedUrl()); + if (e == null) { + e = new TerminologyResourceEntry(); + e.sources.add(source); + e.valueSet = vs; + entries.put(vs.getVersionedUrl(), e); + source = vs.getSourcePackage().getVID()+":"+vs.getVersionedUrl(); + for (ConceptSetComponent inc : vs.getCompose().getInclude()) { + for (CanonicalType v : inc.getValueSet()) { + if (v.hasValue()) { + processValueSet(v.primitiveValue(), source); + } + } + } + for (ConceptSetComponent inc : vs.getCompose().getExclude()) { + for (CanonicalType v : inc.getValueSet()) { + if (v.hasValue()) { + processValueSet(v.primitiveValue(), source); + } + } + } + } else { + e.sources.add(source); + } + + } else { + System.out.println("Unable to resolve value set "+valueSet); + } + + } + + private NpmPackage load() throws IOException { + FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager.Builder().build(); + NpmPackage npm = pcm.loadPackage(packageId); + if (context == null) { + String v = npm.fhirVersion(); + NpmPackage core = pcm.loadPackage(VersionUtilities.packageForVersion(v)); + NpmPackage tho = pcm.loadPackage("hl7.terminology"); + System.out.println("Load FHIR from "+core.name()+"#"+core.version()); + SimpleWorkerContext ctxt = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).fromPackage(core); + TerminologyClientFactory factory = new TerminologyClientFactory(ctxt.getVersion()); + ctxt.connectToTSServer(factory, "http://tx.fhir.org", ctxt.getUserAgent(), null, true); + var loader = new IgLoader(pcm, ctxt, ctxt.getVersion()); + loader.loadPackage(tho, true); + loader.loadPackage(npm, true); + context = ctxt; + } else { + var loader = new IgLoader(pcm, (SimpleWorkerContext) context, context.getVersion()); + loader.loadPackage(npm, true); + } + context.setExpansionParameters(expansionParameters); + cu = new ContextUtilities(context); + return npm; + } + +} diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxServiceTestHelper.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxServiceTestHelper.java index 576b2aa50..cb8a0040d 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxServiceTestHelper.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxServiceTestHelper.java @@ -5,6 +5,7 @@ import org.hl7.fhir.r5.formats.JsonParser; import org.hl7.fhir.r5.context.IWorkerContext; import org.hl7.fhir.r5.formats.IParser; import org.hl7.fhir.r5.model.*; +import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent; import org.hl7.fhir.r5.terminologies.utilities.TerminologyServiceErrorClass; import org.hl7.fhir.r5.terminologies.utilities.ValidationResult; import org.hl7.fhir.r5.test.utils.CompareUtilities; @@ -66,6 +67,11 @@ public class TxServiceTestHelper { if (p.hasParameter("activeOnly") && "true".equals(p.getParameterString("activeOnly"))) { options = options.setActiveOnly(true); } + for (ParametersParameterComponent pp : p.getParameter()) { + if (Utilities.existsInList(pp.getName(), "valueset-version", "system-version", "force-system-version", "default-system-version")) { + context.getExpansionParameters().getParameter().add(pp); + } + } context.getExpansionParameters().clearParameters("includeAlternateCodes"); for (Parameters.ParametersParameterComponent pp : p.getParameter()) { if ("includeAlternateCodes".equals(pp.getName())) { diff --git a/org.hl7.fhir.validation/src/main/resources/help/tx-pack.txt b/org.hl7.fhir.validation/src/main/resources/help/tx-pack.txt new file mode 100644 index 000000000..ca8d46fed --- /dev/null +++ b/org.hl7.fhir.validation/src/main/resources/help/tx-pack.txt @@ -0,0 +1,11 @@ +You can use the validator to generate a package containing all the terminology resources for +an Implementation Guide. To do this, you must provide a specific parameter: + + -tx-pack {package-id} + +-tx-pack requires the parameter -output. All parameters: + +- output {file|folder}: a named file or folder. If it's a file, it must end with .tgz or .zip +- format xml may be used to specify xml instead of json. +- scope ig|igs|all - which to include value sets etc from dependent IGs or core as well +- expansion-parameters {file} - specifies the expansion parameters to use - this can supply fixed versions for code systems and value sets diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java index 516602914..b029f4f00 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java @@ -86,6 +86,9 @@ public class ValidatorCliTests { @Spy CodeGenTask codeGenTask; + @Spy + TxPackTask txPackTask; + @Spy ScanTask scanTask = new ScanTask() { @Override @@ -122,6 +125,7 @@ public class ValidatorCliTests { transformTask, versionTask, codeGenTask, + txPackTask, //validate is the default validateTask );