rework r4 RDF tests to skip fewer examples (#2159)

+ relative URL resolution in RDFParser
+ test non-DomainResources in RDFParserTest
~ factor/organize RDFParserTest
~ avoid EXTERNAL shapes (for now) in shex-java
- disable the bundle-response.json example as it is apparently malformed
+ added urn:-guard to PreResourceStateHl7Org.wereBack ID update

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/ParserState.java

@@ -1260,10 +1260,12 @@ class ParserState<T> {
 				String versionId = elem.getMeta().getVersionId();
 				if (StringUtils.isBlank(elem.getIdElement().getIdPart())) {
 					// Resource has no ID
-				} else if (StringUtils.isNotBlank(versionId)) {
-					elem.getIdElement().setValue(resourceName + "/" + elem.getIdElement().getIdPart() + "/_history/" + versionId);
-				} else {
-					elem.getIdElement().setValue(resourceName + "/" + elem.getIdElement().getIdPart());
+				} else if (!elem.getIdElement().getIdPart().startsWith("urn:")) {
+					if (StringUtils.isNotBlank(versionId)) {
+						elem.getIdElement().setValue(resourceName + "/" + elem.getIdElement().getIdPart() + "/_history/" + versionId);
+					} else {
+						elem.getIdElement().setValue(resourceName + "/" + elem.getIdElement().getIdPart());
+					}
 				}
 			}
 		}

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/RDFParser.java

@@ -29,6 +29,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.jena.datatypes.xsd.XSDDatatype;
 import org.apache.jena.rdf.model.*;
 import org.apache.jena.riot.Lang;
+import org.apache.jena.riot.system.IRIResolver;
 import org.apache.jena.vocabulary.RDF;
 import org.hl7.fhir.instance.model.api.*;
 
@@ -171,10 +172,14 @@ public class RDFParser extends BaseParser {
 		if (!uriBase.endsWith("/")) {
 			uriBase = uriBase + "/";
 		}
-		String resourceUri = uriBase + resource.getIdElement().toUnqualified();
 
 		if (parentResource == null) {
-			parentResource = rdfModel.getResource(resourceUri);
+			if (!resource.getIdElement().toUnqualified().hasIdPart()) {
+				parentResource = rdfModel.getResource(null);
+			} else {
+				String resourceUri = IRIResolver.resolve(resource.getIdElement().toUnqualified().toString(), uriBase).toString();
+				parentResource = rdfModel.getResource(resourceUri);
+			}
 			// If the resource already exists and has statements, return that existing resource.
 			if (parentResource != null && parentResource.listProperties().toList().size() > 0) {
 				return parentResource;

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/parser/RDFParserTest.java

@@ -9,15 +9,14 @@ import fr.inria.lille.shexjava.schema.parsing.GenParser;
 import fr.inria.lille.shexjava.validation.RecursiveValidation;
 import fr.inria.lille.shexjava.validation.ValidationAlgorithm;
 import org.apache.commons.rdf.api.Graph;
-import org.apache.commons.rdf.api.IRI;
+import org.apache.commons.rdf.api.RDFTerm;
 import org.apache.commons.rdf.rdf4j.RDF4J;
 import org.eclipse.rdf4j.model.Model;
-import org.eclipse.rdf4j.model.impl.SimpleIRI;
 import org.eclipse.rdf4j.rio.RDFFormat;
 import org.eclipse.rdf4j.rio.Rio;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.Base;
-import org.hl7.fhir.r4.model.DomainResource;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -25,8 +24,10 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.springframework.core.io.support.ResourcePatternResolver;
 
+import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.StringReader;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -53,6 +54,15 @@ public class RDFParserTest extends BaseTest {
 		fhirSchema = GenParser.parseSchema(schemaFile, Collections.emptyList());
 	}
 
+	// If we can't round-trip JSON, we skip the Turtle round-trip test.
+	private static ArrayList<String> jsonRoundTripErrors = new ArrayList<String>();
+	@AfterAll
+	static void reportJsonRoundTripErrors() {
+		System.out.println(jsonRoundTripErrors.size() + " tests disqualified because of JSON round-trip errors");
+		for (String e : jsonRoundTripErrors)
+			System.out.println(e);
+	}
+
 	/**
 	 * This test method has a method source for each JSON file in the resources/rdf-test-input directory (see #getInputFiles).
 	 * Each input file is expected to be a JSON representation of an R4 FHIR resource.
@@ -62,71 +72,48 @@ public class RDFParserTest extends BaseTest {
 	 * 3. Perform a graph validation on the resulting RDF using ShEx and ShEx-java -- ensure validation passed
 	 * 4. Parse the RDF string into the HAPI object model -- ensure resource instance is not null
 	 * 5. Perform deep equals comparison of JSON-originated instance and RDF-originated instance -- ensure equality
-	 * @param inputFile -- path to resource file to be tested
+	 * @param referenceFilePath -- path to resource file to be tested
 	 * @throws IOException -- thrown when parsing RDF string into graph model
 	 */
 	@ParameterizedTest
 	@MethodSource("getInputFiles")
-	public void testRDFRoundTrip(String inputFile) throws IOException {
-		FileInputStream inputStream = new FileInputStream(inputFile);
-		IBaseResource resource;
-		String resourceType;
-		// Parse JSON input as Resource
-		resource = ourCtx.newJsonParser().parseResource(inputStream);
-		assertNotNull(resource);
-		resourceType = resource.fhirType();
-
-		// Write the resource out to an RDF String
-		String rdfContent = ourCtx.newRDFParser().encodeResourceToString(resource);
-		assertNotNull(rdfContent);
+	public void testRDFRoundTrip(String referenceFilePath) throws IOException {
+		String referenceFileName = referenceFilePath.substring(referenceFilePath.lastIndexOf("/")+1);
+		IBaseResource referenceResource = parseJson(new FileInputStream(referenceFilePath));
+		String referenceJson = serializeJson(ourCtx, referenceResource);
 
 		// Perform ShEx validation on RDF
-		RDF4J factory = new RDF4J();
-		GlobalFactory.RDFFactory = factory; //set the global factory used in shexjava
+		String turtleString = serializeRdf(ourCtx, referenceResource);
+		validateRdf(turtleString, referenceFileName, referenceResource);
 
-		// load the model
-		String baseIRI = "http://a.example.shex/";
-		Model data = Rio.parse(new StringReader(rdfContent), baseIRI, RDFFormat.TURTLE);
+		// If we can round-trip JSON
+		IBaseResource viaJsonResource = parseJson(new ByteArrayInputStream(referenceJson.getBytes()));
+		if (((Base)viaJsonResource).equalsDeep((Base)referenceResource)) {
 
-		String rootSubjectIri = null;
-		for (org.eclipse.rdf4j.model.Resource resourceStream : data.subjects()) {
-			if (resourceStream instanceof SimpleIRI) {
-				Model filteredModel = data.filter(resourceStream, factory.getValueFactory().createIRI(NODE_ROLE_IRI), factory.getValueFactory().createIRI(TREE_ROOT_IRI), (org.eclipse.rdf4j.model.Resource)null);
-				if (filteredModel != null && filteredModel.subjects().size() == 1) {
-					Optional<org.eclipse.rdf4j.model.Resource> rootResource = filteredModel.subjects().stream().findFirst();
-					if (rootResource.isPresent()) {
-						rootSubjectIri = rootResource.get().stringValue();
-						break;
-					}
+			// Parse RDF content as resource
+			IBaseResource viaTurtleResource = parseRdf(ourCtx, new StringReader(turtleString));
+			assertNotNull(viaTurtleResource);
 
-				}
+			// Compare original JSON-based resource against RDF-based resource
+			String viaTurtleJson = serializeJson(ourCtx, viaTurtleResource);
+			if (!((Base)viaTurtleResource).equalsDeep((Base)referenceResource)) {
+				String failMessage = referenceFileName + ": failed to round-trip Turtle ";
+				if (referenceJson.equals(viaTurtleJson))
+					throw new Error(failMessage
+						+ "\nttl: " + turtleString
+						+ "\nexp: " + referenceJson);
+				else
+					assertEquals(referenceJson, viaTurtleJson, failMessage + "\nttl: " + turtleString);
 			}
-		}
-
-		// create the graph
-		Graph dataGraph = factory.asGraph(data);
-
-		// choose focus node and shapelabel
-		IRI focusNode = factory.createIRI(rootSubjectIri);
-		Label shapeLabel = new Label(factory.createIRI(FHIR_SHAPE_PREFIX + resourceType));
-
-		ValidationAlgorithm validation = new RecursiveValidation(fhirSchema, dataGraph);
-		validation.validate(focusNode, shapeLabel);
-		boolean result = validation.getTyping().isConformant(focusNode, shapeLabel);
-		assertTrue(result);
-
-		// Parse RDF content as resource
-		IBaseResource parsedResource = ourCtx.newRDFParser().parseResource(new StringReader(rdfContent));
-		assertNotNull(parsedResource);
-
-		// Compare original JSON-based resource against RDF-based resource
-		if (parsedResource instanceof DomainResource) {
-			// This is a hack because this initializes the collection if it is empty
-			((DomainResource) parsedResource).getContained();
-			boolean deepEquals = ((Base)parsedResource).equalsDeep((Base)resource);
-			assertTrue(deepEquals);
 		} else {
-			ourLog.warn("Input JSON did not yield a DomainResource");
+			String gotString = serializeJson(ourCtx, viaJsonResource);
+			String skipMessage = referenceFileName + ": failed to round-trip JSON" +
+				(referenceJson.equals(gotString)
+					? "\ngot: " + gotString + "\nexp: " + referenceJson
+					: "\nsome inequality not visible in: " + referenceJson);
+			System.out.println(referenceFileName + " skipped");
+			// Specific messages are printed at end of run.
+			jsonRoundTripErrors.add(skipMessage);
 		}
 	}
 
@@ -135,11 +122,98 @@ public class RDFParserTest extends BaseTest {
 		List<String> resourceList = new ArrayList<>();
 		ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(cl);
 		Resource[] resources = resolver.getResources("classpath:rdf-test-input/*.json") ;
-		for (Resource resource: resources){
+		for (Resource resource: resources)
 			resourceList.add(resource.getFile().getPath());
-		}
 
 		return resourceList.stream();
 	}
 
+	// JSON functions
+	public IBaseResource parseJson(InputStream inputStream) {
+		IParser refParser = ourCtx.newJsonParser();
+		refParser.setStripVersionsFromReferences(false);
+		// parser.setDontStripVersionsFromReferencesAtPaths();
+		IBaseResource ret = refParser.parseResource(inputStream);
+		assertNotNull(ret);
+		return ret;
+	}
+
+	public String serializeJson(FhirContext ctx, IBaseResource resource) {
+		IParser jsonParser = ctx.newJsonParser();
+		jsonParser.setStripVersionsFromReferences(false);
+		String ret = jsonParser.encodeResourceToString(resource);
+		assertNotNull(ret);
+		return ret;
+	}
+
+	// Rdf (Turtle) functions
+	public IBaseResource parseRdf(FhirContext ctx, StringReader inputStream) {
+		IParser refParser = ctx.newRDFParser();
+		IBaseResource ret = refParser.parseResource(inputStream);
+		assertNotNull(ret);
+		return ret;
+	}
+
+	public String serializeRdf(FhirContext ctx, IBaseResource resource) {
+		IParser rdfParser = ourCtx.newRDFParser();
+		rdfParser.setStripVersionsFromReferences(false);
+		rdfParser.setServerBaseUrl("http://a.example/fhir/");
+		String ret = rdfParser.encodeResourceToString(resource);
+		assertNotNull(ret);
+		return ret;
+	}
+
+	public void validateRdf(String rdfContent, String referenceFileName, IBaseResource referenceResource) throws IOException {
+		String baseIRI = "http://a.example/shex/";
+		RDF4J factory = new RDF4J();
+		GlobalFactory.RDFFactory = factory; //set the global factory used in shexjava
+		Model data = Rio.parse(new StringReader(rdfContent), baseIRI, RDFFormat.TURTLE);
+		FixedShapeMapEntry fixedMapEntry = new FixedShapeMapEntry(factory, data, referenceResource.fhirType(), baseIRI);
+		Graph dataGraph = factory.asGraph(data); // create the graph
+		ValidationAlgorithm validation = new RecursiveValidation(fhirSchema, dataGraph);
+		validation.validate(fixedMapEntry.node, fixedMapEntry.shape);
+		boolean result = validation.getTyping().isConformant(fixedMapEntry.node, fixedMapEntry.shape);
+		assertTrue(result,
+			   referenceFileName + ": failed to validate " + fixedMapEntry
+			   + "\n" + referenceFileName
+			   + "\n" + rdfContent
+			   );
+	}
+
+	// Shape Expressions functions
+	class FixedShapeMapEntry {
+		RDFTerm node;
+		Label shape;
+
+		FixedShapeMapEntry(RDF4J factory, Model data, String resourceType, String baseIRI) {
+			String rootSubjectIri = null;
+			// StmtIterator i = data.listStatements();
+			for (org.eclipse.rdf4j.model.Resource resourceStream : data.subjects()) {
+//				if (resourceStream instanceof SimpleIRI) {
+					Model filteredModel = data.filter(resourceStream, factory.getValueFactory().createIRI(NODE_ROLE_IRI), factory.getValueFactory().createIRI(TREE_ROOT_IRI), (org.eclipse.rdf4j.model.Resource)null);
+					if (filteredModel != null && filteredModel.subjects().size() == 1) {
+						Optional<org.eclipse.rdf4j.model.Resource> rootResource = filteredModel.subjects().stream().findFirst();
+						if (rootResource.isPresent()) {
+							rootSubjectIri = rootResource.get().stringValue();
+							break;
+						}
+
+					}
+//				}
+			}
+
+			// choose focus node and shapelabel
+			this.node = rootSubjectIri.indexOf(":") == -1
+				? factory.createBlankNode(rootSubjectIri)
+			 	: factory.createIRI(rootSubjectIri);
+			Label shapeLabel = new Label(factory.createIRI(FHIR_SHAPE_PREFIX + resourceType));
+//			this.node = focusNode;
+			this.shape = shapeLabel;
+		}
+
+		public String toString() {
+			return "<" + node.toString() + ">@" + shape.toPrettyString();
+		}
+	}
+
 }

hapi-fhir-structures-r4/src/test/resources/rdf-test-input/DISABLED/README.md

@@ -0,0 +1,5 @@
+# Why is this disabled?
+
+| file | reason |
+| - | - |
+| [bundle-response](bundle-response.json) | [entry\[0\].response.outcome.issue](bundle-response.json#L50-L61) is not in [R4 Resource](http://hl7.org/fhir/resource.html#Resource) |

hapi-fhir-structures-r4/src/test/resources/rdf-validation/fhir-r4.shex

@@ -13267,7 +13267,7 @@ fhirvs:contact-point-use ["home" "work" "temp" "old" "mobile"]
 fhirvs:immunization-status ["completed" "entered-in-error" "not-done"]
 
 # This value set includes all possible codes from BCP-13 (http://tools.ietf.org/html/bcp13)
-fhirvs:mimetypes EXTERNAL
+fhirvs:mimetypes .
 
 # The use of an address.
 fhirvs:address-use ["home" "work" "temp" "old" "billing"]