Fix #477 - Gracefully handle unexpected elements starting with _

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/ErrorHandlerAdapter.java

@@ -1,5 +1,7 @@
 package ca.uhn.fhir.parser;
 
+import ca.uhn.fhir.parser.json.JsonLikeValue.ValueType;
+
 /*
  * #%L
  * HAPI FHIR - Core Library
@@ -30,6 +32,16 @@ public class ErrorHandlerAdapter implements IParserErrorHandler {
 		// NOP
 	}
 
+	@Override
+	public void incorrectJsonType(IParseLocation theLocation, String theElementName, ValueType theExpected, ValueType theFound) {
+		// NOP
+	}
+
+	@Override
+	public void missingRequiredElement(IParseLocation theLocation, String theElementName) {
+		// NOP
+	}
+
 	@Override
 	public void unexpectedRepeatingElement(IParseLocation theLocation, String theElementName) {
 		// NOP
@@ -50,9 +62,4 @@ public class ErrorHandlerAdapter implements IParserErrorHandler {
 		// NOP
 	}
 
-	@Override
-	public void missingRequiredElement(IParseLocation theLocation, String theElementName) {
-		// NOP
-	}
-
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/IParserErrorHandler.java

@@ -1,6 +1,6 @@
 package ca.uhn.fhir.parser;
 
-import ca.uhn.fhir.parser.IParserErrorHandler.IParseLocation;
+import ca.uhn.fhir.parser.json.JsonLikeValue.ValueType;
 
 /*
  * #%L
@@ -37,6 +37,19 @@ public interface IParserErrorHandler {
 	 */
 	void containedResourceWithNoId(IParseLocation theLocation);
 
+	/**
+	 * Invoked if the wrong type of element is found while parsing JSON. For example if a given element is
+	 * expected to be a JSON Object and is a JSON array
+	 * @param theLocation
+	 *           The location in the document. Note that this may be <code>null</code> as the ParseLocation feature is experimental. Use with caution, as the API may change. 
+	 * @param theElementName
+	 *           The name of the element that was found.
+	 * @param theFound The datatype that was found at this location
+	 * @param theExpected The datatype that was expected at this location
+	 * @since 2.2
+	 */
+	void incorrectJsonType(IParseLocation theLocation, String theElementName, ValueType theExpected, ValueType theFound);
+
 	/**
 	 * Resource was missing a required element
 	 * 

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/JsonParser.java

@@ -54,6 +54,7 @@ import ca.uhn.fhir.parser.json.JsonLikeObject;
 import ca.uhn.fhir.parser.json.JsonLikeStructure;
 import ca.uhn.fhir.parser.json.JsonLikeValue;
 import ca.uhn.fhir.parser.json.JsonLikeWriter;
+import ca.uhn.fhir.parser.json.JsonLikeValue.ValueType;
 import ca.uhn.fhir.rest.server.EncodingEnum;
 import ca.uhn.fhir.util.ElementUtil;
 
@@ -1278,6 +1279,11 @@ public class JsonParser extends BaseParser implements IJsonLikeParser {
 			if (alternateVal != null) {
 				handledUnderscoreNames++;
 			}
+			
+			if (alternateVal != null && alternateVal.isObject() == false) {
+				getErrorHandler().incorrectJsonType(null, alternateName, ValueType.OBJECT, alternateVal.getJsonType());
+				alternateVal = null;
+			}
 
 			parseChildren(theState, nextName, nextVal, alternateVal, alternateName);
 

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/LenientErrorHandler.java

@@ -1,7 +1,7 @@
 package ca.uhn.fhir.parser;
 
 import ca.uhn.fhir.context.FhirContext;
-import ca.uhn.fhir.parser.IParserErrorHandler.IParseLocation;
+import ca.uhn.fhir.parser.json.JsonLikeValue.ValueType;
 
 /*
  * #%L
@@ -52,16 +52,23 @@ public class LenientErrorHandler implements IParserErrorHandler {
 	}
 
 	@Override
-	public void unknownElement(IParseLocation theLocation, String theElementName) {
+	public void containedResourceWithNoId(IParseLocation theLocation) {
 		if (myLogErrors) {
-			ourLog.warn("Unknown element '{}' found while parsing", theElementName);
+			ourLog.warn("Resource has contained child resource with no ID");
 		}
 	}
 
 	@Override
-	public void unknownAttribute(IParseLocation theLocation, String theElementName) {
+	public void incorrectJsonType(IParseLocation theLocation, String theElementName, ValueType theExpected, ValueType theFound) {
 		if (myLogErrors) {
-			ourLog.warn("Unknown attribute '{}' found while parsing", theElementName);
+			ourLog.warn("Found incorrect type for element {} - Expected {} and found {}", theElementName, theExpected.name(), theFound.name());
+		}
+	}
+
+	@Override
+	public void missingRequiredElement(IParseLocation theLocation, String theElementName) {
+		if (myLogErrors) {
+			ourLog.warn("Resource is missing required element: {}", theElementName);
 		}
 	}
 
@@ -73,9 +80,16 @@ public class LenientErrorHandler implements IParserErrorHandler {
 	}
 
 	@Override
-	public void containedResourceWithNoId(IParseLocation theLocation) {
+	public void unknownAttribute(IParseLocation theLocation, String theElementName) {
 		if (myLogErrors) {
-			ourLog.warn("Resource has contained child resource with no ID");
+			ourLog.warn("Unknown attribute '{}' found while parsing", theElementName);
+		}
+	}
+
+	@Override
+	public void unknownElement(IParseLocation theLocation, String theElementName) {
+		if (myLogErrors) {
+			ourLog.warn("Unknown element '{}' found while parsing", theElementName);
 		}
 	}
 
@@ -86,11 +100,4 @@ public class LenientErrorHandler implements IParserErrorHandler {
 		}
 	}
 
-	@Override
-	public void missingRequiredElement(IParseLocation theLocation, String theElementName) {
-		if (myLogErrors) {
-			ourLog.warn("Resource is missing required element: {}", theElementName);
-		}
-	}
-
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/StrictErrorHandler.java

@@ -1,6 +1,7 @@
 package ca.uhn.fhir.parser;
 
 import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.parser.json.JsonLikeValue.ValueType;
 
 /*
  * #%L
@@ -70,5 +71,10 @@ public class StrictErrorHandler implements IParserErrorHandler {
 		throw new DataFormatException(b.toString());
 	}
 
+	@Override
+	public void incorrectJsonType(IParseLocation theLocation, String theElementName, ValueType theExpected, ValueType theFound) {
+		throw new DataFormatException("Found incorrect type for element " + theElementName + " - Expected " + theExpected.name() + " and found " + theFound.name());
+	}
+
 
 }

hapi-fhir-jpaserver-uhnfhirtest/derby_maintenance.txt

@@ -3,3 +3,27 @@ CALL SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE( 'SA', 'HFJ_SEARCH_RESULT', 0, 0, 1
 
 
 dd if=/dev/urandom of=/opt/glassfish/tmp.tmp bs=1M count=500
+
+# Reindex
+curl -H "Authorization: Bearer " "http://fhirtest.uhn.ca/baseDstu3/\$mark-all-resources-for-reindexing"
+
+
+# Delete all errored resources
+update hfj_res_ver set forced_id_pid = null where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+update hfj_resource set forced_id_pid = null where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_res_ver where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_forced_id where resource_pid in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_res_link where src_resource_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_res_link where target_resource_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_coords where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_date where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_number where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_quantity where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_string where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_token where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_spidx_uri where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_res_tag where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_search_result where resource_pid in (select res_id from hfj_resource where sp_index_status = 2);
+delete from hfj_resource where res_id in (select res_id from hfj_resource where sp_index_status = 2);
+
+

hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/parser/ErrorHandlerTest.java

@@ -32,6 +32,7 @@ public class ErrorHandlerTest {
 		new ErrorHandlerAdapter().containedResourceWithNoId(null);
 		new ErrorHandlerAdapter().unknownReference(null, null);
 		new ErrorHandlerAdapter().missingRequiredElement(null, null);
+		new ErrorHandlerAdapter().incorrectJsonType(null, null, null, null);
 	}
 
 	@Test
@@ -41,6 +42,7 @@ public class ErrorHandlerTest {
 		new LenientErrorHandler().unknownElement(null, null);
 		new LenientErrorHandler().containedResourceWithNoId(null);
 		new LenientErrorHandler().unknownReference(null, null);
+		new LenientErrorHandler().incorrectJsonType(null, null, null, null);
 	}
 
 	@Test(expected = DataFormatException.class)
@@ -68,4 +70,9 @@ public class ErrorHandlerTest {
 		new StrictErrorHandler().unknownReference(null, null);
 	}
 
+	@Test(expected = DataFormatException.class)
+	public void testStrictMethods6() {
+		new StrictErrorHandler().incorrectJsonType(null, null, null, null);
+	}
+
 }

hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/parser/JsonParserDstu3Test.java

@@ -12,76 +12,42 @@ import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.UUID;
 
 import org.apache.commons.io.IOUtils;
 import org.hamcrest.Matchers;
 import org.hamcrest.core.StringContains;
+import org.hl7.fhir.dstu3.model.*;
 import org.hl7.fhir.dstu3.model.Address.AddressUse;
 import org.hl7.fhir.dstu3.model.Address.AddressUseEnumFactory;
-import org.hl7.fhir.dstu3.model.Attachment;
-import org.hl7.fhir.dstu3.model.AuditEvent;
-import org.hl7.fhir.dstu3.model.Basic;
-import org.hl7.fhir.dstu3.model.Binary;
-import org.hl7.fhir.dstu3.model.Bundle;
 import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.dstu3.model.Bundle.BundleType;
-import org.hl7.fhir.dstu3.model.Coding;
-import org.hl7.fhir.dstu3.model.Communication;
-import org.hl7.fhir.dstu3.model.Condition;
 import org.hl7.fhir.dstu3.model.Condition.ConditionVerificationStatus;
-import org.hl7.fhir.dstu3.model.Conformance;
 import org.hl7.fhir.dstu3.model.Conformance.UnknownContentCode;
-import org.hl7.fhir.dstu3.model.DateTimeType;
-import org.hl7.fhir.dstu3.model.DateType;
-import org.hl7.fhir.dstu3.model.DecimalType;
-import org.hl7.fhir.dstu3.model.DiagnosticReport;
-import org.hl7.fhir.dstu3.model.EnumFactory;
 import org.hl7.fhir.dstu3.model.Enumeration;
 import org.hl7.fhir.dstu3.model.Enumerations.AdministrativeGender;
-import org.hl7.fhir.dstu3.model.ExplanationOfBenefit;
-import org.hl7.fhir.dstu3.model.Extension;
-import org.hl7.fhir.dstu3.model.HumanName;
-import org.hl7.fhir.dstu3.model.IdType;
 import org.hl7.fhir.dstu3.model.Identifier.IdentifierUse;
-import org.hl7.fhir.dstu3.model.Linkage;
-import org.hl7.fhir.dstu3.model.Medication;
-import org.hl7.fhir.dstu3.model.MedicationRequest;
-import org.hl7.fhir.dstu3.model.Observation;
 import org.hl7.fhir.dstu3.model.Observation.ObservationStatus;
-import org.hl7.fhir.dstu3.model.Parameters;
-import org.hl7.fhir.dstu3.model.Patient;
-import org.hl7.fhir.dstu3.model.PrimitiveType;
-import org.hl7.fhir.dstu3.model.Quantity;
-import org.hl7.fhir.dstu3.model.QuestionnaireResponse;
-import org.hl7.fhir.dstu3.model.Reference;
-import org.hl7.fhir.dstu3.model.RelatedPerson;
-import org.hl7.fhir.dstu3.model.SampledData;
-import org.hl7.fhir.dstu3.model.SimpleQuantity;
-import org.hl7.fhir.dstu3.model.StringType;
-import org.hl7.fhir.dstu3.model.UriType;
 import org.hl7.fhir.utilities.xhtml.XhtmlNode;
-import org.junit.After;
+import org.junit.*;
-import org.junit.AfterClass;
+import org.mockito.ArgumentCaptor;
-import org.junit.Assert;
+import org.mockito.Mockito;
-import org.junit.Ignore;
-import org.junit.Test;
 
 import com.google.common.collect.Sets;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
+import ca.uhn.fhir.parser.IParserErrorHandler.IParseLocation;
 import ca.uhn.fhir.parser.PatientWithExtendedContactDstu3.CustomContactComponent;
 import ca.uhn.fhir.parser.XmlParserDstu3Test.TestPatientFor327;
+import ca.uhn.fhir.parser.json.JsonLikeValue.ValueType;
 import ca.uhn.fhir.rest.server.Constants;
 import ca.uhn.fhir.util.TestUtil;
 import ca.uhn.fhir.validation.FhirValidator;
@@ -98,6 +64,31 @@ public class JsonParserDstu3Test {
 	public void after() {
 		ourCtx.setNarrativeGenerator(null);
 	}
+
+	/**
+	 * See #477
+	 */
+	@Test
+	public void testUnexpectedElementsWithUnderscoreAtStartOfName() throws Exception {
+		String input = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bug477.json"), StandardCharsets.UTF_8);
+		
+		IParserErrorHandler errorHandler = mock(IParserErrorHandler.class);
+		
+		IParser p = ourCtx.newJsonParser();
+		p.setParserErrorHandler(errorHandler);
+		
+		Patient parsed = p.parseResource(Patient.class, input);
+		assertEquals("1", parsed.getIdElement().getIdPart());
+		
+		ArgumentCaptor<String> elementName = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<ValueType> expected = ArgumentCaptor.forClass(ValueType.class);
+		ArgumentCaptor<ValueType> actual = ArgumentCaptor.forClass(ValueType.class);
+		verify(errorHandler, times(1)).incorrectJsonType(Mockito.any(IParseLocation.class),elementName.capture(), expected.capture(), actual.capture());
+		
+		assertEquals("_id", elementName.getAllValues().get(0));
+		assertEquals(ValueType.OBJECT, expected.getAllValues().get(0));
+		assertEquals(ValueType.SCALAR, actual.getAllValues().get(0));
+	}
 	
 
 	@Test
@@ -241,9 +232,9 @@ public class JsonParserDstu3Test {
 
 		List<UriType> gotLabels = parsed.getMeta().getProfile();
 		assertEquals(2, gotLabels.size());
-		UriType label = (UriType) gotLabels.get(0);
+		UriType label = gotLabels.get(0);
 		assertEquals("http://foo/Profile1", label.getValue());
-		label = (UriType) gotLabels.get(1);
+		label = gotLabels.get(1);
 		assertEquals("http://foo/Profile2", label.getValue());
 
 		List<Coding> tagList = parsed.getMeta().getTag();
@@ -368,13 +359,13 @@ public class JsonParserDstu3Test {
 
 		assertEquals(2, gotLabels.size());
 
-		Coding label = (Coding) gotLabels.get(0);
+		Coding label = gotLabels.get(0);
 		assertEquals("SYSTEM1", label.getSystem());
 		assertEquals("CODE1", label.getCode());
 		assertEquals("DISPLAY1", label.getDisplay());
 		assertEquals("VERSION1", label.getVersion());
 
-		label = (Coding) gotLabels.get(1);
+		label = gotLabels.get(1);
 		assertEquals("SYSTEM2", label.getSystem());
 		assertEquals("CODE2", label.getCode());
 		assertEquals("DISPLAY2", label.getDisplay());
@@ -1108,7 +1099,7 @@ public class JsonParserDstu3Test {
 		ExplanationOfBenefit eob = ourCtx.newJsonParser().parseResource(ExplanationOfBenefit.class, input);
 		assertEquals(Reference.class, eob.getCoverage().getCoverage().getClass());
 
-		Reference coverage = (Reference) eob.getCoverage().getCoverage();
+		Reference coverage = eob.getCoverage().getCoverage();
 		assertEquals("Coverage/123", coverage.getReference());
 	}
 
@@ -1192,7 +1183,7 @@ public class JsonParserDstu3Test {
 		Patient parsed = ourCtx.newJsonParser().parseResource(Patient.class, input);
 		XhtmlNode div = parsed.getText().getDiv();
 
-		assertEquals("<xhtml:div xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><xhtml:img src=\"foo\"/>@fhirabend</xhtml:div>", parsed.getText().getDiv().getValueAsString());
+		assertEquals("<xhtml:div xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><xhtml:img src=\"foo\"/>@fhirabend</xhtml:div>", div.getValueAsString());
 
 		String encoded = ourCtx.newXmlParser().encodeResourceToString(parsed);
 		assertEquals("<Patient xmlns=\"http://hl7.org/fhir\"><text><xhtml:div xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><xhtml:img src=\"foo\"/>@fhirabend</xhtml:div></text></Patient>", encoded);
@@ -1212,7 +1203,7 @@ public class JsonParserDstu3Test {
 	@Test
 	@Ignore
 	public void testParseAndEncodeBundle() throws Exception {
-		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bundle-example.json"));
+		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bundle-example.json"), StandardCharsets.UTF_8);
 
 		Bundle parsed = ourCtx.newXmlParser().parseResource(Bundle.class, content);
 		assertEquals("Bundle/example/_history/1", parsed.getIdElement().getValue());
@@ -1261,7 +1252,7 @@ public class JsonParserDstu3Test {
 	@Test
 	@Ignore
 	public void testParseAndEncodeBundleFromXmlToJson() throws Exception {
-		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bundle-example2.xml"));
+		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bundle-example2.xml"), StandardCharsets.UTF_8);
 
 		Bundle parsed = ourCtx.newXmlParser().parseResource(Bundle.class, content);
 
@@ -1286,7 +1277,7 @@ public class JsonParserDstu3Test {
 	@Test
 	@Ignore
 	public void testParseAndEncodeBundleNewStyle() throws Exception {
-		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bundle-example.json"));
+		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/bundle-example.json"), StandardCharsets.UTF_8);
 
 		Bundle parsed = ourCtx.newJsonParser().parseResource(Bundle.class, content);
 		assertEquals("Bundle/example/_history/1", parsed.getIdElement().getValue());
@@ -1413,7 +1404,7 @@ public class JsonParserDstu3Test {
 	}
 
 	@Test
-	public void testParseAndEncodeComments() throws IOException {
+	public void testParseAndEncodeComments() {
 		//@formatter:off
 		String input = "{\n" + 
 				"  \"resourceType\": \"Patient\",\n" + 
@@ -1501,7 +1492,7 @@ public class JsonParserDstu3Test {
 	 */
 	@Test
 	public void testParseCommunicationWithThreeTypes() throws IOException {
-		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/tara-test.json"));
+		String content = IOUtils.toString(JsonParserDstu3Test.class.getResourceAsStream("/tara-test.json"), StandardCharsets.UTF_8);
 		Communication comm = ourCtx.newJsonParser().parseResource(Communication.class, content);
 
 		assertEquals(3, comm.getPayload().size());
@@ -1553,7 +1544,7 @@ public class JsonParserDstu3Test {
 	 */
 	@Test
 	public void testParseExtensionWithId() throws Exception {
-		String input = IOUtils.toString(getClass().getResourceAsStream("/json-edge-case-modified-335.json"));
+		String input = IOUtils.toString(getClass().getResourceAsStream("/json-edge-case-modified-335.json"), StandardCharsets.UTF_8);
 
 		Patient p = ourCtx.newJsonParser().parseResource(Patient.class, input);
 		StringType family1 = p.getContact().get(0).getName().getFamily().get(1);

hapi-fhir-structures-dstu3/src/test/resources/bug477.json

@@ -0,0 +1,47 @@
+{
+	"_id": "580e0a706f65e89c12a2b3ce",
+	"id": "Patient/1",
+	"resourceType": "Patient",
+	"__v": 0,
+	"link": [],
+	"careProvider": [],
+	"communication": [],
+	"animal": {
+		"genderStatus": {
+			"coding": []
+		},
+		"breed": {
+			"coding": []
+		},
+		"species": {
+			"coding": []
+		}
+	},
+	"contact": [],
+	"photo": [],
+	"maritalStatus": {
+		"coding": []
+	},
+	"address": [],
+	"telecom": [],
+	"name": [{
+		"_id": "580e0a706f65e89c12a2b3ca",
+		"suffix": [],
+		"prefix": [],
+		"given": [
+			"荣魁"
+		],
+		"family": [
+			"顾"
+		]
+	}],
+	"identifier": [{
+		"system": "urn:oid:2.16.840.1.113883.2.23.11.4.3.98",
+		"value": "278668",
+		"_id": "580e0a706f65e89c12a2b3cb"
+	}],
+	"meta": {
+		"versionId": "2",
+		"lastUpdated": "2016-10-24T21:19:44"
+	}
+}