validation documentation improvements, and batch loading

This commit is contained in:
Grahame Grieve 2019-11-09 07:54:16 +11:00
parent cb70afed0c
commit 0fcb97aa2f
7 changed files with 191 additions and 114 deletions

View File

@ -23,6 +23,7 @@ package org.hl7.fhir.r4.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.UUID;
@ -30,113 +31,155 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.r4.formats.IParser;
import org.hl7.fhir.r4.formats.JsonParser;
import org.hl7.fhir.r4.formats.XmlParser;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.r4.model.Bundle.BundleType;
import org.hl7.fhir.r4.model.Bundle.HTTPVerb;
import org.hl7.fhir.r4.model.Resource;
import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
import org.hl7.fhir.r4.utils.client.ToolingClientLogger;
import org.hl7.fhir.utilities.IniFile;
import org.hl7.fhir.utilities.Utilities;
public class BatchLoader {
public static void main(String[] args) throws IOException, Exception {
if (args.length < 4) {
System.out.println("Batch uploader takes 4 parameters in order: server base url, file/folder to upload, xml/json, and batch size");
if (args.length < 3) {
System.out.println("Batch uploader takes 3 parameters in order: server base url, file/folder to upload, and batch size");
} else {
String server = args[0];
String file = args[1];
IParser p = new JsonParser(); // args[2].equals("json") ? new JsonParser() : new XmlParser();
int size = Integer.parseInt(args[3]);
size = 500;
int size = Integer.parseInt(args[2]);
if (file.endsWith(".xml")) {
throw new FHIRException("Unimplemented file type "+file);
} else if (file.endsWith(".json")) {
throw new FHIRException("Unimplemented file type "+file);
} else if (file.endsWith(".zip")) {
LoadZipFile(server, file, p, size, 0, -1);
// } else if (file.endsWith(".zip")) {
// LoadZipFile(server, file, p, size, 0, -1);
} else if (new File(file).isDirectory()) {
LoadDirectory(server, file, p, size);
LoadDirectory(server, file, size);
} else
throw new FHIRException("Unknown file type "+file);
}
}
private static void LoadDirectory(String server, String file, IParser p, int size) throws IOException, Exception {
// LoadZipFile(server, Utilities.path(file, "Patient.json.zip"), p, size, 1000, -1);
// LoadZipFile(server, Utilities.path(file, "Binary.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "DocumentReference.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Encounter.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Organization.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Procedure.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "AllergyIntolerance.json.zip"), p, size, 1500, -1);
// LoadZipFile(server, Utilities.path(file, "Condition.json.zip"), p, size, 0, -1);
LoadZipFile(server, Utilities.path(file, "Immunization.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "MedicationStatement.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Observation-res.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Observation-sh.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Observation-vs.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "Observation-gen.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "List.json.zip"), p, size, 6500, -1);
// LoadZipFile(server, Utilities.path(file, "List-res.json.zip"), p, size, 0, -1);
// LoadZipFile(server, Utilities.path(file, "List-vs.json.zip"), p, size, 0, -1);
private static void LoadDirectory(String server, String folder, int size) throws IOException, Exception {
System.out.print("Connecting to "+server+".. ");
FHIRToolingClient client = new FHIRToolingClient(server);
System.out.println("Done");
IniFile ini = new IniFile(Utilities.path(folder, "batch-load-progress.ini"));
for (File f : new File(folder).listFiles()) {
if (f.getName().endsWith(".json") || f.getName().endsWith(".xml")) {
if (!ini.getBooleanProperty("finished", f.getName())) {
sendFile(client, f, size, ini);
}
}
}
}
private static void LoadZipFile(String server, String file, IParser p, int size, int start, int end) throws IOException, Exception {
System.out.println("Load Zip file "+file);
Bundle b = new Bundle();
b.setType(BundleType.COLLECTION);
b.setId(UUID.randomUUID().toString().toLowerCase());
ZipInputStream zip = new ZipInputStream(new FileInputStream(file));
ZipEntry entry;
while((entry = zip.getNextEntry())!=null)
{
try {
Resource r = p.parse(zip);
b.addEntry().setResource(r);
} catch (Exception e) {
throw new Exception("Error parsing "+entry.getName()+": "+e.getMessage(), e);
}
}
loadBundle(server, b, size, start, end);
}
private static void sendFile(FHIRToolingClient client, File f, int size, IniFile ini) throws FHIRFormatError, FileNotFoundException, IOException {
System.out.print("Loading "+f.getName()+".. ");
IParser parser = f.getName().endsWith(".json") ? new JsonParser() : new XmlParser();
Resource res = parser.parse(new FileInputStream(f));
System.out.println("Done. Size = "+size);
if (res instanceof Bundle) {
Bundle bnd = (Bundle) res;
int cursor = ini.hasProperty("progress", f.getName()) ? ini.getIntegerProperty("progress", f.getName()) : 0;
while (cursor < bnd.getEntry().size()) {
Bundle bt = new Bundle();
bt.setType(BundleType.BATCH);
bt.setId(UUID.randomUUID().toString().toLowerCase());
for (int i = cursor; i < Math.min(bnd.getEntry().size(), cursor+size); i++) {
BundleEntryComponent be = bt.addEntry();
be.setResource(bnd.getEntry().get(i).getResource());
be.getRequest().setMethod(HTTPVerb.PUT);
be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
}
System.out.print(f.getName()+" ("+cursor+"/"+bnd.getEntry().size()+"): ");
long ms = System.currentTimeMillis();
Bundle resp = client.transaction(bt);
private static int loadBundle(String server, Bundle b, int size, int start, int end) throws URISyntaxException {
System.out.println("Post to "+server+". size = "+Integer.toString(size)+", start = "+Integer.toString(start)+", total = "+Integer.toString(b.getEntry().size()));
FHIRToolingClient client = new FHIRToolingClient(server);
int c = start;
if (end == -1)
end = b.getEntry().size();
while (c < end) {
Bundle bt = new Bundle();
bt.setType(BundleType.BATCH);
bt.setId(UUID.randomUUID().toString().toLowerCase());
for (int i = c; i < Math.min(b.getEntry().size(), c+size); i++) {
BundleEntryComponent be = bt.addEntry();
be.setResource(b.getEntry().get(i).getResource());
be.getRequest().setMethod(HTTPVerb.PUT);
be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
}
System.out.print(" posting..");
long ms = System.currentTimeMillis();
Bundle resp = client.transaction(bt);
for (int i = 0; i < resp.getEntry().size(); i++) {
BundleEntryComponent t = resp.getEntry().get(i);
if (!t.getResponse().getStatus().startsWith("2")) {
System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
return c+i;
}
}
c = c + size;
System.out.println(" ..done: "+Integer.toString(c)+". ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
}
System.out.println(" done");
return c;
}
cursor = cursor+size;
for (int i = 0; i < resp.getEntry().size(); i++) {
BundleEntryComponent t = resp.getEntry().get(i);
if (!t.getResponse().getStatus().startsWith("2")) {
System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
cursor = cursor+i-1;
break;
}
}
System.out.println(" .. done: ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
ini.setIntegerProperty("progress", f.getName(), cursor, null);
ini.save();
}
ini.setBooleanProperty("finished", f.getName(), true, null);
ini.save();
} else {
client.update(res);
ini.setBooleanProperty("finished", f.getName(), true, null);
ini.save();
}
}
//
// private static void LoadZipFile(String server, String file, IParser p, int size, int start, int end) throws IOException, Exception {
// System.out.println("Load Zip file "+file);
// Bundle b = new Bundle();
// b.setType(BundleType.COLLECTION);
// b.setId(UUID.randomUUID().toString().toLowerCase());
// ZipInputStream zip = new ZipInputStream(new FileInputStream(file));
// ZipEntry entry;
// while((entry = zip.getNextEntry())!=null)
// {
// try {
// Resource r = p.parse(zip);
// b.addEntry().setResource(r);
// } catch (Exception e) {
// throw new Exception("Error parsing "+entry.getName()+": "+e.getMessage(), e);
// }
// }
// loadBundle(server, b, size, start, end);
// }
//
//
// private static int loadBundle(String server, Bundle b, int size, int start, int end) throws URISyntaxException {
// System.out.println("Post to "+server+". size = "+Integer.toString(size)+", start = "+Integer.toString(start)+", total = "+Integer.toString(b.getEntry().size()));
// FHIRToolingClient client = new FHIRToolingClient(server);
// int c = start;
// if (end == -1)
// end = b.getEntry().size();
// while (c < end) {
// Bundle bt = new Bundle();
// bt.setType(BundleType.BATCH);
// bt.setId(UUID.randomUUID().toString().toLowerCase());
// for (int i = c; i < Math.min(b.getEntry().size(), c+size); i++) {
// BundleEntryComponent be = bt.addEntry();
// be.setResource(b.getEntry().get(i).getResource());
// be.getRequest().setMethod(HTTPVerb.PUT);
// be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
// }
// System.out.print(" posting..");
// long ms = System.currentTimeMillis();
// Bundle resp = client.transaction(bt);
//
// for (int i = 0; i < resp.getEntry().size(); i++) {
// BundleEntryComponent t = resp.getEntry().get(i);
// if (!t.getResponse().getStatus().startsWith("2")) {
// System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
// return c+i;
// }
// }
// c = c + size;
// System.out.println(" ..done: "+Integer.toString(c)+". ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
// }
// System.out.println(" done");
// return c;
// }
}

View File

@ -263,35 +263,64 @@ public class FHIRToolingClient {
return (T) bnd.getEntry().get(0).getResource();
}
//
// public <T extends Resource> T update(Class<T> resourceClass, T resource, String id) {
// ResourceRequest<T> result = null;
// try {
// List<Header> headers = null;
// result = utils.issuePutRequest(resourceAddress.resolveGetUriFromResourceClassAndId(resourceClass, id),utils.getResourceAsByteArray(resource, false, isJson(getPreferredResourceFormat())), getPreferredResourceFormat(), headers, proxy);
// result.addErrorStatus(410);//gone
// result.addErrorStatus(404);//unknown
// result.addErrorStatus(405);
// result.addErrorStatus(422);//Unprocessable Entity
// result.addSuccessStatus(200);
// result.addSuccessStatus(201);
// if(result.isUnsuccessfulRequest()) {
// throw new EFhirClientException("Server returned error code " + result.getHttpStatus(), (OperationOutcome)result.getPayload());
// }
// } catch(Exception e) {
// throw new EFhirClientException("An error has occurred while trying to update this resource", e);
// }
// // TODO oe 26.1.2015 could be made nicer if only OperationOutcome locationheader is returned with an operationOutcome would be returned (and not the resource also) we make another read
// try {
// OperationOutcome operationOutcome = (OperationOutcome)result.getPayload();
// ResourceAddress.ResourceVersionedIdentifier resVersionedIdentifier = ResourceAddress.parseCreateLocation(result.getLocation());
// return this.vread(resourceClass, resVersionedIdentifier.getId(),resVersionedIdentifier.getVersionId());
// } catch(ClassCastException e) {
// // if we fall throught we have the correct type already in the create
// }
//
// return result.getPayload();
// }
public Resource update(Resource resource) {
ResourceRequest<Resource> result = null;
try {
List<Header> headers = null;
result = utils.issuePutRequest(resourceAddress.resolveGetUriFromResourceClassAndId(resource.getClass(), resource.getId()),utils.getResourceAsByteArray(resource, false, isJson(getPreferredResourceFormat())), getPreferredResourceFormat(), headers);
result.addErrorStatus(410);//gone
result.addErrorStatus(404);//unknown
result.addErrorStatus(405);
result.addErrorStatus(422);//Unprocessable Entity
result.addSuccessStatus(200);
result.addSuccessStatus(201);
if(result.isUnsuccessfulRequest()) {
throw new EFhirClientException("Server returned error code " + result.getHttpStatus(), (OperationOutcome)result.getPayload());
}
} catch(Exception e) {
throw new EFhirClientException("An error has occurred while trying to update this resource", e);
}
// TODO oe 26.1.2015 could be made nicer if only OperationOutcome locationheader is returned with an operationOutcome would be returned (and not the resource also) we make another read
try {
OperationOutcome operationOutcome = (OperationOutcome)result.getPayload();
ResourceAddress.ResourceVersionedIdentifier resVersionedIdentifier = ResourceAddress.parseCreateLocation(result.getLocation());
return this.vread(resource.getClass(), resVersionedIdentifier.getId(),resVersionedIdentifier.getVersionId());
} catch(ClassCastException e) {
// if we fall throught we have the correct type already in the create
}
return result.getPayload();
}
public <T extends Resource> T update(Class<T> resourceClass, T resource, String id) {
ResourceRequest<T> result = null;
try {
List<Header> headers = null;
result = utils.issuePutRequest(resourceAddress.resolveGetUriFromResourceClassAndId(resourceClass, id),utils.getResourceAsByteArray(resource, false, isJson(getPreferredResourceFormat())), getPreferredResourceFormat(), headers);
result.addErrorStatus(410);//gone
result.addErrorStatus(404);//unknown
result.addErrorStatus(405);
result.addErrorStatus(422);//Unprocessable Entity
result.addSuccessStatus(200);
result.addSuccessStatus(201);
if(result.isUnsuccessfulRequest()) {
throw new EFhirClientException("Server returned error code " + result.getHttpStatus(), (OperationOutcome)result.getPayload());
}
} catch(Exception e) {
throw new EFhirClientException("An error has occurred while trying to update this resource", e);
}
// TODO oe 26.1.2015 could be made nicer if only OperationOutcome locationheader is returned with an operationOutcome would be returned (and not the resource also) we make another read
try {
OperationOutcome operationOutcome = (OperationOutcome)result.getPayload();
ResourceAddress.ResourceVersionedIdentifier resVersionedIdentifier = ResourceAddress.parseCreateLocation(result.getLocation());
return this.vread(resourceClass, resVersionedIdentifier.getId(),resVersionedIdentifier.getVersionId());
} catch(ClassCastException e) {
// if we fall throught we have the correct type already in the create
}
return result.getPayload();
}
//
// public <T extends Resource> boolean delete(Class<T> resourceClass, String id) {

View File

@ -58,6 +58,7 @@ public class Constants {
public final static String NS_SYSTEM_TYPE = "http://hl7.org/fhirpath/System.";
public final static String VERSION = "4.1.0";
public final static String VERSION_MM = "4.1";
public final static String BUILD_ID = "2ce21320c7";
public final static String DATE = "Thu Oct 17 19:56:09 AEDT 2019";
public final static String URI_REGEX = "((http|https)://([A-Za-z0-9\\\\\\.\\:\\%\\$]*\\/)*)?(Account|ActivityDefinition|AdministrableProductDefinition|AdverseEvent|AllergyIntolerance|Appointment|AppointmentResponse|AuditEvent|Basic|Binary|BiologicallyDerivedProduct|BodyStructure|Bundle|CapabilityStatement|CapabilityStatement2|CarePlan|CareTeam|CatalogEntry|ChargeItem|ChargeItemDefinition|Claim|ClaimResponse|ClinicalImpression|ClinicalUseIssue|CodeSystem|Communication|CommunicationRequest|CompartmentDefinition|Composition|ConceptMap|Condition|ConditionDefinition|Consent|Contract|Coverage|CoverageEligibilityRequest|CoverageEligibilityResponse|DetectedIssue|Device|DeviceDefinition|DeviceMetric|DeviceRequest|DeviceUseStatement|DiagnosticReport|DocumentManifest|DocumentReference|Encounter|Endpoint|EnrollmentRequest|EnrollmentResponse|EpisodeOfCare|EventDefinition|Evidence|EvidenceVariable|ExampleScenario|ExplanationOfBenefit|FamilyMemberHistory|Flag|Goal|GraphDefinition|Group|GuidanceResponse|HealthcareService|ImagingStudy|Immunization|ImmunizationEvaluation|ImmunizationRecommendation|ImplementationGuide|Ingredient|InsurancePlan|Invoice|Library|Linkage|List|Location|ManufacturedItemDefinition|Measure|MeasureReport|Medication|MedicationAdministration|MedicationDispense|MedicationKnowledge|MedicationRequest|MedicationUsage|MedicinalProductDefinition|MessageDefinition|MessageHeader|MolecularSequence|NamingSystem|NutritionIntake|NutritionOrder|Observation|ObservationDefinition|OperationDefinition|OperationOutcome|Organization|OrganizationAffiliation|PackagedProductDefinition|Patient|PaymentNotice|PaymentReconciliation|Person|PlanDefinition|Practitioner|PractitionerRole|Procedure|Provenance|Questionnaire|QuestionnaireResponse|RegulatedAuthorization|RelatedPerson|RequestGroup|ResearchStudy|ResearchSubject|RiskAssessment|Schedule|SearchParameter|ServiceRequest|Slot|Specimen|SpecimenDefinition|StructureDefinition|StructureMap|Subscription|Substance|SubstanceDefinition|SubstanceNucleicAcid|SubstancePolymer|SubstanceProtein|SubstanceReferenceInformation|SubstanceSourceMaterial|SupplyDelivery|SupplyRequest|Task|TerminologyCapabilities|TestReport|TestScript|Topic|ValueSet|VerificationResult|VisionPrescription)\\/[A-Za-z0-9\\-\\.]{1,64}(\\/_history\\/[A-Za-z0-9\\-\\.]{1,64})?";

View File

@ -1526,4 +1526,8 @@ public final class IniFile
}
return false;
}
public boolean hasProperty(String section, String name) {
return getStringProperty(section, name) != null;
}
}

View File

@ -160,8 +160,8 @@ public class Validator {
System.out.println("");
System.out.println("The FHIR validation tool validates a FHIR resource or bundle.");
System.out.println("Schema and schematron checking is performed, then some additional checks are performed. ");
System.out.println("* XML & Json (FHIR versions 1.0, 1.4, 3.0, 3.4)");
System.out.println("* Turtle (FHIR versions 3.0, 3.4)");
System.out.println("* XML & Json (FHIR versions 1.0, 1.4, 3.0, 4.0, "+Constants.VERSION_MM+")");
System.out.println("* Turtle (FHIR versions 3.0, 4.0, "+Constants.VERSION_MM+")");
System.out.println("");
System.out.println("If requested, instances will also be verified against the appropriate schema");
System.out.println("W3C XML Schema, JSON schema or ShEx, as appropriate");
@ -264,8 +264,8 @@ public class Validator {
if (v == null) v = Constants.VERSION;
else if ("1.0".equals(v)) v = "1.0.2";
else if ("1.4".equals(v)) v = "1.4.0";
else if ("3.0".equals(v)) v = "3.0.1";
else if ("4.0".equals(v)) v = "4.0.0";
else if ("3.0".equals(v)) v = "3.0.2";
else if ("4.0".equals(v)) v = "4.0.1";
else if (v.startsWith(Constants.VERSION)) v = Constants.VERSION;
String definitions = "hl7.fhir.core#"+v;
System.out.println("Loading (v = "+v+", tx server http://tx.fhir.org)");

View File

@ -17,7 +17,7 @@
<properties>
<hapi_fhir_version>4.1.0-SNAPSHOT</hapi_fhir_version>
<validator_test_case_version>1.0.3-SNAPSHOT</validator_test_case_version>
<validator_test_case_version>1.0.4-SNAPSHOT</validator_test_case_version>
</properties>
<artifactId>org.hl7.fhir.core</artifactId>

View File

@ -1,7 +1,7 @@
@echo off
set oldver=4.0.30
set newver=4.0.31
set oldver=4.0.31
set newver=4.0.32
echo ..
echo =====================================================================