From f1d5676425a52a3ae07d1cc4080e5662756157dc Mon Sep 17 00:00:00 2001 From: Grahame Grieve Date: Sun, 16 Apr 2023 21:56:41 +1000 Subject: [PATCH] run txTests from validator --- .../java/org/hl7/fhir/utilities/Servers.java | 3 +- .../org/hl7/fhir/validation/ValidatorCli.java | 35 +++-- .../hl7/fhir/validation/cli/utils/Params.java | 1 + .../hl7/fhir/validation/special/TxTester.java | 138 ++++++++++++++++-- .../ExternalTerminologyServiceTests.java | 1 + 5 files changed, 156 insertions(+), 22 deletions(-) diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Servers.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Servers.java index df46f9b4e..fa83255ad 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Servers.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Servers.java @@ -5,9 +5,10 @@ public class Servers { public static final String TX_SERVER_PROD = "http://tx.fhir.org"; public static final String TX_SERVER_DEV = "http://tx-dev.fhir.org"; public static final String TX_SERVER_LOCAL = "http://local.fhir.org"; + public static final String TX_SERVER_LOCAL2 = "http://local.fhir.org:8090"; public static boolean isTxFhirOrg(String s) { - return Utilities.existsInList(s.replace("https://", "http://"), TX_SERVER_PROD, TX_SERVER_DEV, TX_SERVER_LOCAL); + return Utilities.startsWithInList(s.replace("https://", "http://"), TX_SERVER_PROD, TX_SERVER_DEV, TX_SERVER_LOCAL); } } diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java index e85f198e5..92476f49f 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidatorCli.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.net.Authenticator; import java.net.PasswordAuthentication; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -84,6 +85,8 @@ import org.hl7.fhir.validation.cli.utils.Display; import org.hl7.fhir.validation.cli.utils.EngineMode; import org.hl7.fhir.validation.cli.utils.Params; import org.hl7.fhir.validation.special.R4R5MapTester; +import org.hl7.fhir.validation.special.TxTester; +import org.hl7.fhir.validation.special.TxTester.InternalTxLoader; import org.hl7.fhir.validation.testexecutor.TestExecutor; import org.hl7.fhir.validation.testexecutor.TestExecutorParams; @@ -135,7 +138,7 @@ public class ValidatorCli { if (destinationDirectoryValid(Params.getParam(args, Params.DESTINATION))) { doLeftRightComparison(args, cliContext, tt); } - } else if (Params.hasParam(args, Params.TEST)) { + } else if (Params.hasParam(args, Params.TEST) || Params.hasParam(args, Params.TX_TESTS)) { parseTestParamsAndExecute(args); } else if (Params.hasParam(args, Params.SPECIAL)) { executeSpecial(args); @@ -208,19 +211,29 @@ public class ValidatorCli { } } - protected static void parseTestParamsAndExecute(String[] args) { - final String testModuleParam = Params.getParam(args, Params.TEST_MODULES); - final String testClassnameFilter = Params.getParam(args, Params.TEST_NAME_FILTER); - final String testCasesDirectory = Params.getParam(args, Params.TEST); - final String txCacheDirectory = Params.getParam(args, Params.TERMINOLOGY_CACHE); - assert TestExecutorParams.isValidModuleParam(testModuleParam) : "Invalid test module param: " + testModuleParam; - final String[] moduleNamesArg = TestExecutorParams.parseModuleParam(testModuleParam); + protected static void parseTestParamsAndExecute(String[] args) throws IOException, URISyntaxException { + if (Params.hasParam(args, Params.TX_TESTS)) { + final String source = Params.getParam(args, Params.SOURCE); + final String output = Params.getParam(args, Params.OUTPUT); + final String version = Params.getParam(args, Params.VERSION); + final String tx = Params.getParam(args, Params.TERMINOLOGY); + final String filter = Params.getParam(args, Params.FILTER); + boolean ok = new TxTester(new InternalTxLoader(source, output)).setOutput(output).execute(tx, version, filter); + System.exit(ok ? 1 : 0); + } else { + final String testModuleParam = Params.getParam(args, Params.TEST_MODULES); + final String testClassnameFilter = Params.getParam(args, Params.TEST_NAME_FILTER); + final String testCasesDirectory = Params.getParam(args, Params.TEST); + final String txCacheDirectory = Params.getParam(args, Params.TERMINOLOGY_CACHE); + assert TestExecutorParams.isValidModuleParam(testModuleParam) : "Invalid test module param: " + testModuleParam; + final String[] moduleNamesArg = TestExecutorParams.parseModuleParam(testModuleParam); - assert TestExecutorParams.isValidClassnameFilterParam(testClassnameFilter) : "Invalid regex for test classname filter: " + testClassnameFilter; + assert TestExecutorParams.isValidClassnameFilterParam(testClassnameFilter) : "Invalid regex for test classname filter: " + testClassnameFilter; - new TestExecutor(moduleNamesArg).executeTests(testClassnameFilter, txCacheDirectory, testCasesDirectory); + new TestExecutor(moduleNamesArg).executeTests(testClassnameFilter, txCacheDirectory, testCasesDirectory); - System.exit(0); + System.exit(0); + } } private static String[] addAdditionalParamsForIpsParam(String[] args) { diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java index f65362e4f..23c5700f9 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/Params.java @@ -57,6 +57,7 @@ public class Params { public static final String CONVERT = "-convert"; public static final String FHIRPATH = "-fhirpath"; public static final String TEST = "-tests"; + public static final String TX_TESTS = "-txTests"; public static final String HELP = "help"; public static final String COMPARE = "-compare"; public static final String SPREADSHEET = "-spreadsheet"; diff --git a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxTester.java b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxTester.java index 11c1d3215..89519f1db 100644 --- a/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxTester.java +++ b/org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/special/TxTester.java @@ -4,11 +4,23 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.sql.Date; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collections; +import java.util.GregorianCalendar; import java.util.List; +import java.util.Locale; +import java.util.TimeZone; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.hl7.fhir.convertors.txClient.TerminologyClientFactory; import org.hl7.fhir.exceptions.DefinitionException; @@ -27,8 +39,10 @@ import org.hl7.fhir.utilities.FhirPublication; import org.hl7.fhir.utilities.TextFile; import org.hl7.fhir.utilities.Utilities; import org.hl7.fhir.utilities.json.JsonException; +import org.hl7.fhir.utilities.json.model.JsonArray; import org.hl7.fhir.utilities.json.model.JsonObject; import org.hl7.fhir.utilities.json.parser.JsonParser; +import org.hl7.fhir.utilities.npm.NpmPackage; public class TxTester { @@ -42,6 +56,7 @@ public class TxTester { private String server; private ITxTesterLoader loader; private String error; + private String output; public TxTester(ITxTesterLoader loader) { @@ -50,30 +65,61 @@ public class TxTester { } public static void main(String[] args) throws Exception { - new TxTester(new InternalLoader(args[0])).execute(args[1], args[2]); + new TxTester(new InternalTxLoader(args[0])).execute(args[1], args[2], args[3]); } - public void execute(String server, String filter) { + public boolean execute(String server, String version, String filter) throws IOException, URISyntaxException { this.server = server; - System.out.println("Run terminology service Tests"); + if (output == null) { + output = Utilities.path("[tmp]", serverId()); + } + System.out.println("Run terminology service Tests"); + System.out.println(" Source for tests: "+loader.describe()); + System.out.println(" Output Directory: "+output); + System.out.println(" Term Service Url: "+server); + if (version != null) { + System.out.println(" Tx FHIR Version: "+version); + } + if (filter != null) { + System.out.println(" Filter Parameter: "+filter); + } + + JsonObject json = new JsonObject(); + json.add("date", new SimpleDateFormat("EEE, MMM d, yyyy HH:mmZ", new Locale("en", "US")).format(Calendar.getInstance().getTime()) + timezone()); try { JsonObject tests = loadTests(); TerminologyClient tx = connectToServer(); boolean ok = checkClient(tx); for (JsonObject suite : tests.getJsonObjects("suites")) { - ok = runSuite(suite, tx, filter) && ok; + ok = runSuite(suite, tx, filter, json.forceArray("suites")) && ok; } + TextFile.stringToFile(JsonParser.compose(json, true), Utilities.path(output, "test-results.json")); if (ok) { System.out.println("Terminology Service Tests all passed"); + return true; } else { System.out.println("Terminology Service Tests did not all pass"); + return false; } } catch (Exception e) { System.out.println("Exception running Terminology Service Tests: "+e.getMessage()); e.printStackTrace(); + return false; } } + + + private String timezone() { + TimeZone tz = TimeZone.getDefault(); + Calendar cal = GregorianCalendar.getInstance(tz); + int offsetInMillis = tz.getOffset(cal.getTimeInMillis()); + + String offset = String.format("%02d:%02d", Math.abs(offsetInMillis / 3600000), Math.abs((offsetInMillis / 60000) % 60)); + offset = (offsetInMillis >= 0 ? "+" : "-") + offset; + + return offset; + } private boolean checkClient(TerminologyClient tx) { tx.getCapabilitiesStatementQuick(); @@ -98,23 +144,31 @@ public class TxTester { TerminologyClient tx = connectToServer(); checkClient(tx); List setup = loadSetupResources(suite); - if (runTest(test, tx, setup, "*")) { + if (runTest(test, tx, setup, "*", null)) { return null; } else { return error; } } - private boolean runSuite(JsonObject suite, TerminologyClient tx, String filter) throws FHIRFormatError, FileNotFoundException, IOException { + private boolean runSuite(JsonObject suite, TerminologyClient tx, String filter, JsonArray output) throws FHIRFormatError, FileNotFoundException, IOException { System.out.println("Group "+suite.asString("name")); + JsonObject outputS = new JsonObject(); + if (output != null) { + output.add(outputS); + } + outputS.add("name", suite.asString("name")); List setup = loadSetupResources(suite); boolean ok = true; for (JsonObject test : suite.getJsonObjects("tests")) { - ok = runTest(test, tx, setup, filter) && ok; + ok = runTest(test, tx, setup, filter, outputS.forceArray("tests")) && ok; } return ok; } - private boolean runTest(JsonObject test, TerminologyClient tx, List setup, String filter) { + private boolean runTest(JsonObject test, TerminologyClient tx, List setup, String filter, JsonArray output) { + JsonObject outputT = new JsonObject(); + output.add(outputT); + outputT.add("name", test.asString("name")); if (Utilities.noString(filter) || filter.equals("*") || test.asString("name").contains(filter)) { System.out.print(" Test "+test.asString("name")+": "); try { @@ -142,6 +196,10 @@ public class TxTester { System.out.println(" "+msg); error = msg; } + outputT.add("status", msg == null ? "pass" : "fail"); + if (msg != null) { + outputT.add("message", msg); + } return msg == null; } catch (Exception e) { System.out.println(" ... Exception: "+e.getMessage()); @@ -151,6 +209,7 @@ public class TxTester { return false; } } else { + outputT.add("status", "ignored"); return true; } } @@ -213,14 +272,73 @@ public class TxTester { return res; } - public static class InternalLoader implements ITxTesterLoader { + public String getOutput() { + return output; + } + + public TxTester setOutput(String output) { + this.output = output; + return this; + } + + public static class InternalTxLoader implements ITxTesterLoader { private String folder; - public InternalLoader(String folder) { + public InternalTxLoader(String folder) { this.folder = folder; } + + public InternalTxLoader(String source, String local) throws IOException { + if (source.startsWith("http://") || source.startsWith("https://")) { + this.folder = Utilities.path(local, "source"); + URL url = new URL(zipUrl(source)); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + InputStream zip = connection.getInputStream(); + unzip(zip); + } else { + this.folder = source; + } + } + + public void unzip(InputStream is) throws IOException { + try (ZipInputStream zipIn = new ZipInputStream(is)) { + for (ZipEntry ze; (ze = zipIn.getNextEntry()) != null; ) { + if (ze.getName().startsWith("fhir-test-cases-master/tx/")) { + Path path = Path.of(Utilities.path(this.folder, ze.getName().substring(26))).normalize(); + String pathString = path.toFile().getAbsolutePath(); + if (!path.startsWith(Path.of(this.folder).normalize())) { + // see: https://snyk.io/research/zip-slip-vulnerability + throw new RuntimeException("Entry with an illegal path: " + ze.getName()); + } + if (ze.isDirectory()) { + Utilities.createDirectory(pathString); + } else { + Utilities.createDirectory(Utilities.getDirectoryForFile(pathString)); + TextFile.streamToFileNoClose(zipIn, pathString); + } + } + } + } + } + + + private String zipUrl(String template) { + if (!template.startsWith("https://github.")) { + throw new FHIRException("Cannot refer to source by URL unless referring to a github repository: "+template); + } else if (Utilities.charCount(template, '/') == 4) { + return Utilities.pathURL(template, "archive", "master.zip"); + } else if (Utilities.charCount(template, '/') == 6) { + String[] p = template.split("\\/"); + return Utilities.pathURL("https://"+p[2], p[3], p[4], "archive", p[6]+".zip"); + } else { + throw new FHIRException("Source syntax in URL referring to a github repository was not understood: "+template); + } + } + + @Override public String describe() { return folder; diff --git a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/terminology/tests/ExternalTerminologyServiceTests.java b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/terminology/tests/ExternalTerminologyServiceTests.java index a7f959812..2df32a3db 100644 --- a/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/terminology/tests/ExternalTerminologyServiceTests.java +++ b/org.hl7.fhir.validation/src/test/java/org/hl7/fhir/terminology/tests/ExternalTerminologyServiceTests.java @@ -48,6 +48,7 @@ public class ExternalTerminologyServiceTests implements ITxTesterLoader { } private static final String SERVER = Servers.TX_SERVER_DEV; +// private static final String SERVER = Servers.TX_SERVER_LOCAL2; @Parameters(name = "{index}: id {0}") public static Iterable data() throws IOException {