Add support for terminology extraction and support for expansion parameters when validating

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/Manager.java

@@ -111,6 +111,14 @@ public class Manager {
       }
       return null;
     }
+
+    public static FhirFormat fromCode(String code) {
+      FhirFormat fmt = getFhirFormat(code);
+      if (fmt == null) {
+        fmt = readFromMimeType(code);
+      } 
+      return fmt;
+    }
   }
   
   public static List<ValidatedFragment> parse(IWorkerContext context, InputStream source, FhirFormat inputFormat) throws FHIRFormatError, DefinitionException, IOException, FHIRException {

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java

@@ -2,6 +2,7 @@ package org.hl7.fhir.validation;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -1295,4 +1296,24 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
     return ReferenceValidationPolicy.IGNORE;
   }
 
+  public void loadExpansionParameters(String expansionParameters) {
+    System.out.println("Load Expansion Parameters: "+expansionParameters);
+    Parameters p = null;
+    try {
+      p = (Parameters) new XmlParser().parse(new FileInputStream(expansionParameters));
+    } catch (Exception e) {
+    }
+    if (p == null) {
+      try {
+        p = (Parameters) new JsonParser().parse(new FileInputStream(expansionParameters));
+      } catch (Exception e) {
+        System.out.println("Unable to load expansion parameters '"+expansionParameters+"' as either xml or json: "+e.getMessage());
+        throw new FHIRException("Unable to load expansion parameters '"+expansionParameters+"' as either xml or json: "+e.getMessage());
+      }
+    }
+    context.setExpansionParameters(p);
+
+
+  }
+
 }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java

@@ -137,6 +137,7 @@ public class ValidatorCli {
       new TransformTask(),
       new VersionTask(),
       new CodeGenTask(),
+      new TxPackTask(),
       defaultCliTask);
   }
 

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/model/CliContext.java

@@ -1,5 +1,6 @@
 package org.hl7.fhir.validation.cli.model;
 
+import java.io.FileInputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -8,6 +9,11 @@ import java.util.Map;
 import java.util.Objects;
 
 import com.google.gson.annotations.SerializedName;
+
+import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
+import org.hl7.fhir.r5.formats.JsonParser;
+import org.hl7.fhir.r5.formats.XmlParser;
+import org.hl7.fhir.r5.model.Parameters;
 import org.hl7.fhir.r5.terminologies.JurisdictionUtilities;
 import org.hl7.fhir.r5.utils.validation.BundleValidationRule;
 import org.hl7.fhir.r5.utils.validation.constants.BestPracticeWarningLevel;
@@ -322,6 +328,16 @@ public class CliContext {
   private
   String advisorFile;
 
+  @JsonProperty("expansionParameters")
+  @SerializedName("expansionParameters")
+  private
+  String expansionParameters;
+
+  @JsonProperty("format")
+  @SerializedName("format")
+  private
+  FhirFormat format;
+
   @SerializedName("baseEngine")
   @JsonProperty("baseEngine")
   public String getBaseEngine() {
@@ -1146,6 +1162,8 @@ public class CliContext {
       Objects.equals(unknownCodeSystemsCauseErrors, that.unknownCodeSystemsCauseErrors) &&
       Objects.equals(noExperimentalContent, that.noExperimentalContent) &&
       Objects.equals(advisorFile, that.advisorFile) &&
+      Objects.equals(expansionParameters, that.expansionParameters) &&
+      Objects.equals(format, that.format) &&
       Objects.equals(watchSettleTime, that.watchSettleTime);
   }
 
@@ -1154,7 +1172,7 @@ public class CliContext {
     return Objects.hash(baseEngine, doNative, extensions, hintAboutNonMustSupport, recursive, doDebug, assumeValidRestReferences, canDoNative, noInternalCaching,
       noExtensibleBindingMessages, noInvariants, displayWarnings, wantInvariantsInMessages, map, output, outputSuffix, htmlOutput, txServer, sv, txLog, txCache, mapLog, lang, srcLang, tgtLang, fhirpath, snomedCT,
       targetVer, packageName, igs, questionnaireMode, level, profiles, options, sources, inputs, mode, locale, locations, crumbTrails, showMessageIds, forPublication, showTimes, allowExampleUrls, outputStyle, jurisdiction, noUnicodeBiDiControlChars,
-      watchMode, watchScanDelay, watchSettleTime, bestPracticeLevel, unknownCodeSystemsCauseErrors, noExperimentalContent, advisorFile, htmlInMarkdownCheck, allowDoubleQuotesInFHIRPath, checkIPSCodes);
+      watchMode, watchScanDelay, watchSettleTime, bestPracticeLevel, unknownCodeSystemsCauseErrors, noExperimentalContent, advisorFile, expansionParameters, format, htmlInMarkdownCheck, allowDoubleQuotesInFHIRPath, checkIPSCodes);
   }
 
   @Override
@@ -1219,6 +1237,8 @@ public class CliContext {
       ", unknownCodeSystemsCauseErrors=" + unknownCodeSystemsCauseErrors +
       ", noExperimentalContent=" + noExperimentalContent +
       ", advisorFile=" + advisorFile +
+      ", expansionParameters=" + expansionParameters +
+      ", format=" + format +
       '}';
   }
 
@@ -1325,5 +1345,28 @@ public class CliContext {
     this.advisorFile = advisorFile;
   }
 
+  @SerializedName("expansionParameters")
+  @JsonProperty("expansionParameters")
+  public String getExpansionParameters() {
+    return expansionParameters;
+  }
+
+  @SerializedName("expansionParameters")
+  @JsonProperty("expansionParameters")
+  public void setExpansionParameters(String expansionParameters) {
+    this.expansionParameters = expansionParameters;
+  }
+
+  @SerializedName("format")
+  @JsonProperty("format")
+  public FhirFormat getFormat() {
+    return format;
+  }
+
+  @SerializedName("format")
+  @JsonProperty("format")
+  public void setFormat(FhirFormat format) {
+    this.format = format;
+  }
 
 }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/TxPackTask.java

@@ -0,0 +1,83 @@
+package org.hl7.fhir.validation.cli.tasks;
+
+import java.io.File;
+import java.io.PrintStream;
+
+import org.hl7.fhir.r5.context.IWorkerContext;
+import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
+import org.hl7.fhir.utilities.TimeTracker;
+import org.hl7.fhir.validation.ValidationEngine;
+import org.hl7.fhir.validation.cli.model.CliContext;
+import org.hl7.fhir.validation.cli.services.ValidationService;
+import org.hl7.fhir.validation.cli.utils.Display;
+import org.hl7.fhir.validation.cli.utils.EngineMode;
+import org.hl7.fhir.validation.special.ExpansionPackageGenerator;
+import org.hl7.fhir.validation.special.ExpansionPackageGenerator.ExpansionPackageGeneratorOutputType;
+import org.hl7.fhir.validation.special.ExpansionPackageGenerator.ExpansionPackageGeneratorScope;
+
+public class TxPackTask extends ValidationEngineTask {
+
+  @Override
+  public String getName() {
+    return "tx-pack";
+  }
+
+  @Override
+  public String getDisplayName() {
+    return "Generate a terminology pack";
+  }
+
+  @Override
+  public boolean isHidden() {
+    return false;
+  }
+
+  @Override
+  public boolean shouldExecuteTask(CliContext cliContext, String[] args) {
+    return cliContext.getMode() == EngineMode.TX_PACK;
+  }
+
+  @Override
+  public void printHelp(PrintStream out) {
+    Display.displayHelpDetails(out,"help/tx-pack.txt");
+  }
+
+  @Override
+  public void executeTask(ValidationService validationService, ValidationEngine validationEngine, CliContext cliContext, String[] args, TimeTracker tt, TimeTracker.Session tts) throws Exception { 
+    String pid = cliContext.getPackageName();
+    boolean json = cliContext.getFormat() != FhirFormat.XML;
+    String output = cliContext.getOutput();
+    File f = new File(output);
+    ExpansionPackageGeneratorOutputType t = ExpansionPackageGeneratorOutputType.FOLDER;
+    if (f.exists() && f.isDirectory()) {
+      t = ExpansionPackageGeneratorOutputType.FOLDER;
+    } else if (output.endsWith(".zip")) {
+      t = ExpansionPackageGeneratorOutputType.ZIP;
+    } else if (output.endsWith(".tgz")) {
+      t = ExpansionPackageGeneratorOutputType.TGZ;
+    } 
+    ExpansionPackageGeneratorScope scope = ExpansionPackageGeneratorScope.IG_ONLY;
+    int c = -1;
+    for (int i = 0; i < args.length; i++) {
+      if ("-scope".equals(args[i])) {
+        c = i;
+      }
+    }
+    if (c < args.length - 1) {
+      switch (args[c+1].toLowerCase()) {
+      case "ig" : scope = ExpansionPackageGeneratorScope.IG_ONLY;
+      case "igs" : scope = ExpansionPackageGeneratorScope.ALL_IGS;
+      case "core" : scope = ExpansionPackageGeneratorScope.EVERYTHING;
+      default: 
+        System.out.println("Unknown scope "+args[c+1]);
+      }
+    }
+    IWorkerContext ctxt = validationEngine.getContext();
+    ExpansionPackageGenerator ep = new ExpansionPackageGenerator().setContext(ctxt).setPackageId(pid).setScope(scope);
+    if (cliContext.getExpansionParameters() != null) {
+      validationEngine.loadExpansionParameters(cliContext.getExpansionParameters());
+    }
+    ep.setOutput(output).setOutputType(t);
+    ep.generateExpansionPackage();
+  }
+}

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/tasks/ValidateTask.java

@@ -51,6 +51,10 @@ public class ValidateTask extends ValidationEngineTask {
 
   @Override
   public void executeTask(ValidationService validationService, ValidationEngine validationEngine, CliContext cliContext, String[] args, TimeTracker tt, TimeTracker.Session tts) throws Exception {
+    if (cliContext.getExpansionParameters() != null) {
+      validationEngine.loadExpansionParameters(cliContext.getExpansionParameters());
+    }
+    
     for (String s : cliContext.getProfiles()) {
       if (!validationEngine.getContext().hasResource(StructureDefinition.class, s) && !validationEngine.getContext().hasResource(ImplementationGuide.class, s)) {
         System.out.println("  Fetch Profile from " + s);

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/EngineMode.java

@@ -14,5 +14,6 @@ public enum EngineMode {
   VERSION,
   RUN_TESTS,
   INSTALL,
-  CODEGEN
+  CODEGEN,
+  TX_PACK
 }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java

@@ -6,6 +6,7 @@ import java.util.Arrays;
 import java.util.Locale;
 
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
 import org.hl7.fhir.r5.terminologies.JurisdictionUtilities;
 import org.hl7.fhir.r5.utils.validation.BundleValidationRule;
 import org.hl7.fhir.r5.utils.validation.constants.BestPracticeWarningLevel;
@@ -45,13 +46,16 @@ public class Params {
   public static final String EXTENSION = "-extension";
   public static final String HINT_ABOUT_NON_MUST_SUPPORT = "-hintAboutNonMustSupport";
   public static final String TO_VERSION = "-to-version";
+  public static final String TX_PACK = "-tx-pack";
   public static final String PACKAGE_NAME = "-package-name";
   public static final String DO_NATIVE = "-do-native";
   public static final String NO_NATIVE = "-no-native";
   public static final String COMPILE = "-compile";
   public static final String CODEGEN = "-codegen";
   public static final String TRANSFORM = "-transform";
+  public static final String FORMAT = "-format";
   public static final String LANG_TRANSFORM = "-lang-transform";
+  public static final String EXP_PARAMS = "-expansion-parameters";
   public static final String NARRATIVE = "-narrative";
   public static final String SNAPSHOT = "-snapshot";
   public static final String INSTALL = "-install";
@@ -330,6 +334,9 @@ public class Params {
       } else if (args[i].equals(PACKAGE_NAME)) {
         cliContext.setPackageName(args[++i]);
         cliContext.setMode(EngineMode.CODEGEN);
+      } else if (args[i].equals(TX_PACK)) {
+        cliContext.setPackageName(args[++i]);
+        cliContext.setMode(EngineMode.TX_PACK);
       } else if (args[i].equals(DO_NATIVE)) {
         cliContext.setCanDoNative(true);
       } else if (args[i].equals(NO_NATIVE)) {
@@ -337,9 +344,13 @@ public class Params {
       } else if (args[i].equals(TRANSFORM)) {
         cliContext.setMap(args[++i]);
         cliContext.setMode(EngineMode.TRANSFORM);
+      } else if (args[i].equals(FORMAT)) {
+        cliContext.setFormat(FhirFormat.fromCode(args[++i]));
       } else if (args[i].equals(LANG_TRANSFORM)) {
         cliContext.setLangTransform(args[++i]);
         cliContext.setMode(EngineMode.LANG_TRANSFORM);
+      } else if (args[i].equals(EXP_PARAMS)) {
+        cliContext.setExpansionParameters(args[++i]);
       } else if (args[i].equals(COMPILE)) {
         cliContext.setMap(args[++i]);
         cliContext.setMode(EngineMode.COMPILE);

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/ExpansionPackageGenerator.java

@@ -0,0 +1,409 @@
+package org.hl7.fhir.validation.special;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.hl7.fhir.convertors.txClient.TerminologyClientFactory;
+import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.r5.context.ContextUtilities;
+import org.hl7.fhir.r5.context.IWorkerContext;
+import org.hl7.fhir.r5.context.SimpleWorkerContext;
+import org.hl7.fhir.r5.formats.IParser.OutputStyle;
+import org.hl7.fhir.r5.formats.JsonParser;
+import org.hl7.fhir.r5.formats.XmlParser;
+import org.hl7.fhir.r5.model.CanonicalType;
+import org.hl7.fhir.r5.model.ElementDefinition;
+import org.hl7.fhir.r5.model.Parameters;
+import org.hl7.fhir.r5.model.StructureDefinition;
+import org.hl7.fhir.r5.model.ValueSet;
+import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
+import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome;
+import org.hl7.fhir.r5.utils.NPMPackageGenerator;
+import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.VersionUtilities;
+import org.hl7.fhir.utilities.ZipGenerator;
+import org.hl7.fhir.utilities.json.model.JsonObject;
+import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
+import org.hl7.fhir.utilities.npm.NpmPackage;
+import org.hl7.fhir.validation.IgLoader;
+import org.hl7.fhir.validation.special.ExpansionPackageGenerator.TerminologyResourceEntry;
+
+/**
+ * Given a package id, and an expansion parameters, 
+ * get a list of all the value sets in the package, optionally with expansions. 
+ * Parameters:
+ * 
+ * - scope: this ig | all igs | all igs + core
+ * - expansions: true | false
+ * - output type: folder | zip | tgz
+ */
+
+public class ExpansionPackageGenerator {
+
+  public static class TerminologyResourceEntry {
+    public ValueSet valueSet;
+    public Set<String> sources = new HashSet<>();
+    public String error;
+  }
+
+  public static void main(String[] args) throws Exception {
+    new ExpansionPackageGenerator()
+      .setPackageId("hl7.fhir.us.davinci-alerts")
+      .setJson(true)
+      .setOutputType(ExpansionPackageGeneratorOutputType.TGZ)
+      .setOutput("/Users/grahamegrieve/temp/vs-output.tgz")
+      .generateExpansionPackage();
+  }
+  public enum ExpansionPackageGeneratorOutputType {
+    FOLDER, ZIP, TGZ
+  }
+
+  public enum ExpansionPackageGeneratorScope {
+    IG_ONLY, ALL_IGS, EVERYTHING
+  }
+
+  private String packageId;
+  private Parameters expansionParameters = new Parameters(); 
+  private ExpansionPackageGeneratorScope scope = ExpansionPackageGeneratorScope.EVERYTHING;
+  private boolean expansions;
+  private String output;
+  private ExpansionPackageGeneratorOutputType outputType;
+  private boolean hierarchical;
+  private boolean json;
+  private IWorkerContext context;
+  
+  
+  public String getPackageId() {
+    return packageId;
+  }
+
+  public ExpansionPackageGenerator setPackageId(String packageId) {
+    this.packageId = packageId;
+    return this;
+  }
+
+  public Parameters getExpansionParameters() {
+    return expansionParameters;
+  }
+
+  public ExpansionPackageGenerator setExpansionParameters(Parameters expansionParameters) {
+    this.expansionParameters = expansionParameters;
+    return this;
+  }
+
+  public ExpansionPackageGeneratorScope getScope() {
+    return scope;
+  }
+
+  public ExpansionPackageGenerator setScope(ExpansionPackageGeneratorScope scope) {
+    this.scope = scope;
+    return this;
+  }
+
+  public boolean isExpansions() {
+    return expansions;
+  }
+
+  public ExpansionPackageGenerator setExpansions(boolean expansions) {
+    this.expansions = expansions;
+    return this;
+  }
+
+  public String getOutput() {
+    return output;
+  }
+
+  public ExpansionPackageGenerator setOutput(String output) {
+    this.output = output;
+    return this;
+  }
+
+  public ExpansionPackageGeneratorOutputType getOutputType() {
+    return outputType;
+  }
+
+  public ExpansionPackageGenerator setOutputType(ExpansionPackageGeneratorOutputType outputType) {
+    this.outputType = outputType;
+    return this;
+  }
+
+  public boolean isHierarchical() {
+    return hierarchical;
+  }
+
+  public ExpansionPackageGenerator setHierarchical(boolean hierarchical) {
+    this.hierarchical = hierarchical;
+    return this;
+  }
+
+  public boolean isJson() {
+    return json;
+  }
+
+  public ExpansionPackageGenerator setJson(boolean json) {
+    this.json = json;
+    return this;
+  }
+
+
+  public IWorkerContext getContext() {
+    return context;
+  }
+
+  public ExpansionPackageGenerator setContext(IWorkerContext context) {
+    this.context = context;
+    return this;
+  }
+
+
+  private ContextUtilities cu;
+  
+  private Map<String, TerminologyResourceEntry> entries = new HashMap<>();
+  
+  public void generateExpansionPackage() throws IOException {
+    if (output == null) {
+      throw new Error("No output");
+    }
+
+    
+    var npm = load();
+    System.out.println("Finding ValueSets");
+    for (String res : npm.listResources("StructureDefinition")) {
+      StructureDefinition sd = (StructureDefinition) new JsonParser().parse(npm.loadResource(res));
+      processSD(sd, npm.id());
+      
+    }
+    System.out.println("Generating Expansions");
+    for (String n : Utilities.sorted(entries.keySet())) {
+      TerminologyResourceEntry e = entries.get(n);
+      try {
+        System.out.print("Generate Expansion for "+n+" ... ");
+        ValueSetExpansionOutcome exp = context.expandVS(e.valueSet, true, hierarchical);
+        if (exp.isOk()) {
+          e.valueSet.setExpansion(exp.getValueset().getExpansion());
+          System.out.println("OK");
+        } else {
+          e.valueSet.setExpansion(null);
+          e.error = exp.getError();
+          System.out.println(exp.getError());
+        }
+      } catch (Exception ex) {
+        System.out.println("Error= "+ex.getMessage());
+        e.error = ex.getMessage();
+      }
+    }
+
+    System.out.println("Producing Output");
+    switch (outputType) {
+    case FOLDER:
+      produceFolder();
+      break;
+    case TGZ:
+      producePackage(npm);
+      break;
+    case ZIP:
+      produceZip();
+      break;
+    default:
+      break;
+    
+    }
+    System.out.println("Done");
+  }
+
+  private void produceZip() throws IOException {
+    ZipGenerator zip = new ZipGenerator(output);
+    Set<String> names = new HashSet<>();
+    names.add("manifest");
+    if (json) {
+      zip.addBytes("manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false);
+    } else {
+      zip.addBytes("manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters), false);
+    }
+    StringBuilder b = new StringBuilder();
+    for (String n : Utilities.sorted(entries.keySet())) {
+      TerminologyResourceEntry e = entries.get(n);
+      String name = e.valueSet.getIdBase();
+      int i = 0;
+      while (names.contains(name)) {
+        i++;
+        name = e.valueSet.getIdBase()+i;
+      }
+      names.add(name);
+      if (e.error == null) {
+        b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n");
+      } else {
+        b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n");
+      }
+      if (json) {
+        zip.addBytes(name+".json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet), false);
+      } else {
+        zip.addBytes(name+".xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet), false);
+      }
+    }
+    zip.addBytes("valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8), false);
+    zip.close();
+  }
+
+  private void producePackage(NpmPackage npm) throws FHIRException, IOException {
+    JsonObject j = new JsonObject();
+    j.add("name", npm.id()+".custom.extensions");
+    j.add("version", npm.version());
+    j.add("tools-version", 3);
+    j.add("type", "Conformance");
+    j.forceArray("fhirVersions").add(npm.fhirVersion());
+    j.forceObject("dependencies").add(VersionUtilities.packageForVersion(npm.fhirVersion()), npm.fhirVersion());
+    
+    NPMPackageGenerator gen = new NPMPackageGenerator(output, j, new Date(), true);
+
+    Set<String> names = new HashSet<>();
+    names.add("manifest");
+    if (json) {
+      gen.addFile("package", "manifest.json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters));
+    } else {
+      gen.addFile("package", "manifest.xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(expansionParameters));
+    }
+    StringBuilder b = new StringBuilder();
+    for (String n : Utilities.sorted(entries.keySet())) {
+      TerminologyResourceEntry e = entries.get(n);
+      String name = e.valueSet.getIdBase();
+      int i = 0;
+      while (names.contains(name)) {
+        i++;
+        name = e.valueSet.getIdBase()+i;
+      }
+      names.add(name);
+      if (e.error == null) {
+        b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n");
+      } else {
+        b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n");
+      }
+      if (json) {
+        gen.addFile("package",name+".json", new JsonParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet));
+      } else {
+        gen.addFile("package",name+".xml", new XmlParser().setOutputStyle(OutputStyle.PRETTY).composeBytes(e.valueSet));
+      }
+    }
+    gen.addFile("other","valuesets.csv", b.toString().getBytes(StandardCharsets.UTF_8));
+    gen.finish();
+  }
+
+  private void produceFolder() throws IOException {
+    Utilities.createDirectory(output);
+    Utilities.clearDirectory(output);
+    Set<String> names = new HashSet<>();
+    names.add("manifest");
+    if (json) {
+      new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.json")), expansionParameters);
+    } else {
+      new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, "manifest.xml")), expansionParameters);
+    }
+    StringBuilder b = new StringBuilder();
+    for (String n : Utilities.sorted(entries.keySet())) {
+      TerminologyResourceEntry e = entries.get(n);
+      String name = e.valueSet.getIdBase();
+      int i = 0;
+      while (names.contains(name)) {
+        i++;
+        name = e.valueSet.getIdBase()+i;
+      }
+      names.add(name);
+      if (e.error == null) {
+        b.append(name+","+n+", , "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n");
+      } else {
+        b.append(name+","+n+", \""+Utilities.escapeCSV(e.error)+"\", "+CommaSeparatedStringBuilder.join(";",e.sources)+"\r\n");
+      }
+      if (json) {
+        new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, name+".json")), e.valueSet);
+      } else {
+        new XmlParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(output, name+".xml")), e.valueSet);
+      }
+    }
+    TextFile.stringToFile(b.toString(), Utilities.path(output, "valuesets.csv"));
+  }
+
+  private void processSD(StructureDefinition sd, String packageId) {
+//    System.out.println("Found Structure: "+sd.getVersionedUrl());
+    for (ElementDefinition ed : sd.getDifferential().getElement()) {
+      if (ed.hasBinding() && ed.getBinding().hasValueSet()) {
+//        TerminologyResourceEntry e = new TerminologyResourceEntry();
+        processValueSet(ed.getBinding().getValueSet(), packageId+":"+sd.getVersionedUrl()+"#"+ed.getId());
+      }
+    }
+    if (scope != ExpansionPackageGeneratorScope.IG_ONLY && sd.getBaseDefinition() != null) {
+      StructureDefinition bsd = context.fetchResource(StructureDefinition.class, sd.getBaseDefinition());
+      if (!bsd.getUrl().startsWith("http://hl7.org/fhir/StructureDefinition") || scope == ExpansionPackageGeneratorScope.EVERYTHING) {
+        processSD(bsd, bsd.getSourcePackage().getVID());
+      }
+    }
+  }
+
+
+  private void processValueSet(String valueSet, String source) {
+    String url = cu.pinValueSet(valueSet);
+    ValueSet vs = context.fetchResource(ValueSet.class, url);
+    if (vs != null) {
+      TerminologyResourceEntry e = entries.get(vs.getVersionedUrl());
+      if (e == null) {
+        e = new TerminologyResourceEntry();
+        e.sources.add(source);
+        e.valueSet = vs;
+        entries.put(vs.getVersionedUrl(), e);
+        source = vs.getSourcePackage().getVID()+":"+vs.getVersionedUrl();
+        for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
+          for (CanonicalType v : inc.getValueSet()) {
+            if (v.hasValue()) {
+              processValueSet(v.primitiveValue(), source);
+            }
+          }
+        }
+        for (ConceptSetComponent inc : vs.getCompose().getExclude()) {
+          for (CanonicalType v : inc.getValueSet()) {
+            if (v.hasValue()) {
+              processValueSet(v.primitiveValue(), source);
+            }
+          }
+        }
+      } else {
+        e.sources.add(source);
+      }
+      
+    } else {
+      System.out.println("Unable to resolve value set "+valueSet);
+    }
+    
+  }
+
+  private NpmPackage load() throws IOException {
+    FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager.Builder().build();
+    NpmPackage npm = pcm.loadPackage(packageId);
+    if (context == null) {
+      String v = npm.fhirVersion();
+      NpmPackage core = pcm.loadPackage(VersionUtilities.packageForVersion(v));
+      NpmPackage tho = pcm.loadPackage("hl7.terminology");
+      System.out.println("Load FHIR from "+core.name()+"#"+core.version());
+      SimpleWorkerContext ctxt = new SimpleWorkerContext.SimpleWorkerContextBuilder().withAllowLoadingDuplicates(true).fromPackage(core);
+      TerminologyClientFactory factory = new TerminologyClientFactory(ctxt.getVersion());
+      ctxt.connectToTSServer(factory, "http://tx.fhir.org", ctxt.getUserAgent(), null, true);
+      var loader = new IgLoader(pcm, ctxt, ctxt.getVersion());
+      loader.loadPackage(tho, true);
+      loader.loadPackage(npm, true);
+      context = ctxt;
+    } else {
+      var loader = new IgLoader(pcm, (SimpleWorkerContext) context, context.getVersion());
+      loader.loadPackage(npm, true);
+    }
+    context.setExpansionParameters(expansionParameters);
+    cu = new ContextUtilities(context);
+    return npm;
+  }
+
+}

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxServiceTestHelper.java

@@ -5,6 +5,7 @@ import org.hl7.fhir.r5.formats.JsonParser;
 import org.hl7.fhir.r5.context.IWorkerContext;
 import org.hl7.fhir.r5.formats.IParser;
 import org.hl7.fhir.r5.model.*;
+import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
 import org.hl7.fhir.r5.terminologies.utilities.TerminologyServiceErrorClass;
 import org.hl7.fhir.r5.terminologies.utilities.ValidationResult;
 import org.hl7.fhir.r5.test.utils.CompareUtilities;
@@ -66,6 +67,11 @@ public class TxServiceTestHelper {
       if (p.hasParameter("activeOnly") && "true".equals(p.getParameterString("activeOnly"))) {
         options = options.setActiveOnly(true);
       }
+      for (ParametersParameterComponent pp : p.getParameter()) {
+        if (Utilities.existsInList(pp.getName(), "valueset-version", "system-version", "force-system-version", "default-system-version")) {
+          context.getExpansionParameters().getParameter().add(pp);
+        }
+      }
       context.getExpansionParameters().clearParameters("includeAlternateCodes");
       for (Parameters.ParametersParameterComponent pp : p.getParameter()) {
         if ("includeAlternateCodes".equals(pp.getName())) {

org.hl7.fhir.validation/src/main/resources/help/tx-pack.txt

@@ -0,0 +1,11 @@
+You can use the validator to generate a package containing all the terminology resources for 
+an Implementation Guide. To do this, you must provide a specific parameter:
+
+ -tx-pack {package-id}
+
+-tx-pack requires the parameter -output. All parameters:
+
+- output {file|folder}: a named file or folder. If it's a file, it must end with .tgz or .zip
+- format xml may be used to specify xml instead of json.
+- scope ig|igs|all - which to include value sets etc from dependent IGs or core as well
+- expansion-parameters {file} - specifies the expansion parameters to use - this can supply fixed versions for code systems and value sets

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/ValidatorCliTests.java

@@ -86,6 +86,9 @@ public class ValidatorCliTests {
   @Spy
   CodeGenTask codeGenTask;
 
+  @Spy
+  TxPackTask txPackTask;
+
   @Spy
   ScanTask scanTask = new ScanTask() {
     @Override
@@ -122,6 +125,7 @@ public class ValidatorCliTests {
           transformTask,
           versionTask,
           codeGenTask,
+          txPackTask,
           //validate is the default
           validateTask
         );