Fix #750 - Elements are not preserved in page requests

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/BaseParser.java

@@ -43,7 +43,7 @@ public abstract class BaseParser implements IParser {
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseParser.class);
 
 	private ContainedResources myContainedResources;
-
+	private boolean myEncodeElementsAppliesToChildResourcesOnly;
 	private FhirContext myContext;
 	private Set<String> myDontEncodeElements;
 	private boolean myDontEncodeElementsIncludesStars;
@@ -556,6 +556,16 @@ public abstract class BaseParser implements IParser {
 			&& theIncludedResource == false;
 	}
 
+	@Override
+	public boolean isEncodeElementsAppliesToChildResourcesOnly() {
+		return myEncodeElementsAppliesToChildResourcesOnly;
+	}
+
+	@Override
+	public void setEncodeElementsAppliesToChildResourcesOnly(boolean theEncodeElementsAppliesToChildResourcesOnly) {
+		myEncodeElementsAppliesToChildResourcesOnly = theEncodeElementsAppliesToChildResourcesOnly;
+	}
+
 	@Override
 	public boolean isOmitResourceId() {
 		return myOmitResourceId;
@@ -1039,7 +1049,13 @@ public abstract class BaseParser implements IParser {
 		}
 
 		private boolean checkIfParentShouldBeEncodedAndBuildPath(StringBuilder thePathBuilder, boolean theStarPass) {
-			return checkIfPathMatchesForEncoding(thePathBuilder, theStarPass, myEncodeElementsAppliesToResourceTypes, myEncodeElements, true);
+			Set<String> encodeElements = myEncodeElements;
+			if (encodeElements != null && encodeElements.isEmpty() == false) {
+				if (isEncodeElementsAppliesToChildResourcesOnly() && !mySubResource) {
+					encodeElements = null;
+				}
+			}
+			return checkIfPathMatchesForEncoding(thePathBuilder, theStarPass, myEncodeElementsAppliesToResourceTypes, encodeElements, true);
 		}
 
 		private boolean checkIfParentShouldNotBeEncodedAndBuildPath(StringBuilder thePathBuilder, boolean theStarPass) {
@@ -1058,6 +1074,9 @@ public abstract class BaseParser implements IParser {
 				} else {
 					thePathBuilder.append(myResDef.getName());
 				}
+				if (theElements == null) {
+					return true;
+				}
 				if (theElements.contains(thePathBuilder.toString())) {
 					return true;
 				}

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/IParser.java

@@ -206,6 +206,22 @@ public interface IParser {
 	 */
 	void setEncodeElements(Set<String> theEncodeElements);
 
+	/**
+	 * If set to <code>true</code> (default is false), the values supplied
+	 * to {@link #setEncodeElements(Set)} will not be applied to the root
+	 * resource (typically a Bundle), but will be applied to any sub-resources
+	 * contained within it (i.e. search result resources in that bundle)
+	 */
+	void setEncodeElementsAppliesToChildResourcesOnly(boolean theEncodeElementsAppliesToChildResourcesOnly);
+
+	/**
+	 * If set to <code>true</code> (default is false), the values supplied
+	 * to {@link #setEncodeElements(Set)} will not be applied to the root
+	 * resource (typically a Bundle), but will be applied to any sub-resources
+	 * contained within it (i.e. search result resources in that bundle)
+	 */
+	boolean isEncodeElementsAppliesToChildResourcesOnly();
+
 	/**
 	 * If provided, tells the parse which resource types to apply {@link #setEncodeElements(Set) encode elements} to. Any
 	 * resource types not specified here will be encoded completely, with no elements excluded.

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/BinaryUtil.java

@@ -19,12 +19,13 @@ public class BinaryUtil {
 	public static IBaseReference getSecurityContext(FhirContext theCtx, IBaseBinary theBinary) {
 		RuntimeResourceDefinition def = theCtx.getResourceDefinition("Binary");
 		BaseRuntimeChildDefinition child = def.getChildByName("securityContext");
-
-		List<IBase> values = child.getAccessor().getValues(theBinary);
 		IBaseReference retVal = null;
+		if (child != null) {
+			List<IBase> values = child.getAccessor().getValues(theBinary);
 			if (values.size() > 0) {
 				retVal = (IBaseReference) values.get(0);
 			}
+		}
 		return retVal;
 	}
 

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/RestfulServerUtils.java

@@ -86,10 +86,41 @@ public class RestfulServerUtils {
 			}
 		}
 		if (elements != null && elements.size() > 0) {
-			Set<String> newElements = new HashSet<String>();
+			Set<String> newElements = new HashSet<>();
 			for (String next : elements) {
 				newElements.add("*." + next);
 			}
+
+			/*
+			 * We try to be smart about what the user is asking for
+			 * when they include an _elements parameter. If we're responding
+			 * to something that returns a Bundle (e.g. a search) we assume
+			 * the elements don't apply to the Bundle itself, unless
+			 * the client has explicitly scoped the Bundle
+			 * (i.e. with Bundle.total or something like that)
+			 */
+			switch (theRequestDetails.getRestOperationType()) {
+				case SEARCH_SYSTEM:
+				case SEARCH_TYPE:
+				case HISTORY_SYSTEM:
+				case HISTORY_TYPE:
+				case HISTORY_INSTANCE:
+				case GET_PAGE:
+					boolean haveExplicitBundleElement = false;
+					for (String next : newElements) {
+						if (next.startsWith("Bundle.")) {
+							haveExplicitBundleElement = true;
+							break;
+						}
+					}
+					if (!haveExplicitBundleElement) {
+						parser.setEncodeElementsAppliesToChildResourcesOnly(true);
+					}
+					break;
+				default:
+					break;
+			}
+
 			parser.setEncodeElements(newElements);
 			parser.setEncodeElementsAppliesToResourceTypes(elementsAppliesTo);
 		}
@@ -147,6 +178,19 @@ public class RestfulServerUtils {
 				b.append(theBundleType.getCode());
 			}
 
+			String paramName = Constants.PARAM_ELEMENTS;
+			String[] params = theRequestParameters.get(paramName);
+			if (params != null) {
+				for (String nextValue : params) {
+					if (isNotBlank(nextValue)) {
+						b.append('&');
+						b.append(paramName);
+						b.append('=');
+						b.append(UrlUtil.escape(nextValue));
+					}
+				}
+			}
+
 			return b.toString();
 		} catch (UnsupportedEncodingException e) {
 			throw new Error("UTF-8 not supported", e);// should not happen
@@ -587,10 +631,12 @@ public class RestfulServerUtils {
 			response.addHeader(Constants.HEADER_CONTENT_DISPOSITION, "Attachment;");
 
 			IBaseReference securityContext = BinaryUtil.getSecurityContext(theServer.getFhirContext(), bin);
+			if (securityContext != null) {
 				String securityContextRef = securityContext.getReferenceElement().getValue();
 				if (isNotBlank(securityContextRef)) {
 					response.addHeader(Constants.HEADER_X_SECURITY_CONTEXT, securityContextRef);
 				}
+			}
 
 			return response.sendAttachmentResponse(bin, theStausCode, contentType);
 		}

hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/method/BaseResourceReturningMethodBinding.java

@@ -332,7 +332,7 @@ public abstract class BaseResourceReturningMethodBinding extends BaseMethodBindi
 		}
 
 		bundleFactory.addRootPropertiesToBundle(theResult.getUuid(), serverBase, theLinkSelf, linkPrev, linkNext, theResult.size(), theBundleType, theResult.getPublished());
-		bundleFactory.addResourcesToBundle(new ArrayList<IBaseResource>(resourceList), theBundleType, serverBase, theServer.getBundleInclusionRule(), theIncludes);
+		bundleFactory.addResourcesToBundle(new ArrayList<>(resourceList), theBundleType, serverBase, theServer.getBundleInclusionRule(), theIncludes);
 
 		if (theServer.getPagingProvider() != null) {
 			int limit;

hapi-fhir-structures-dstu2.1/src/test/java/ca/uhn/fhir/parser/JsonParserDstu2_1Test.java

@@ -1499,7 +1499,7 @@ public class JsonParserDstu2_1Test {
 
 		String val = ourCtx.newJsonParser().encodeResourceToString(patient);
 
-		String expected = "{\"resourceType\":\"Binary\",\"id\":\"11\",\"contentType\":\"foo\",\"content\":\"AQIDBA==\"}";
+		String expected = "{\"resourceType\":\"Binary\",\"id\":\"11\",\"meta\":{\"versionId\":\"22\"},\"contentType\":\"foo\",\"content\":\"AQIDBA==\"}";
 		ourLog.info("Expected: {}", expected);
 		ourLog.info("Actual  : {}", val);
 		assertEquals(expected, val);

hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/parser/XmlParserDstu3Test.java

@@ -1,41 +1,26 @@
 package ca.uhn.fhir.parser;
 
-import static org.hamcrest.Matchers.contains;
+import ca.uhn.fhir.context.FhirContext;
-import static org.hamcrest.Matchers.containsString;
+import ca.uhn.fhir.model.api.annotation.Child;
-import static org.hamcrest.Matchers.empty;
+import ca.uhn.fhir.model.api.annotation.ResourceDef;
-import static org.hamcrest.Matchers.not;
+import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
-import static org.hamcrest.Matchers.startsWith;
+import ca.uhn.fhir.parser.FooMessageHeaderWithExplicitField.FooMessageSourceComponent;
-import static org.hamcrest.Matchers.stringContainsInOrder;
+import ca.uhn.fhir.parser.IParserErrorHandler.IParseLocation;
-import static org.junit.Assert.assertArrayEquals;
+import ca.uhn.fhir.parser.PatientWithCustomCompositeExtension.FooParentExtension;
-import static org.junit.Assert.assertEquals;
+import ca.uhn.fhir.util.TestUtil;
-import static org.junit.Assert.assertFalse;
+import com.google.common.collect.Sets;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.hamcrest.collection.IsEmptyCollection;
 import org.hamcrest.core.StringContains;
 import org.hamcrest.text.StringContainsInOrder;
-import org.hl7.fhir.dstu3.model.*;
 import org.hl7.fhir.dstu3.model.Address.AddressUse;
 import org.hl7.fhir.dstu3.model.Address.AddressUseEnumFactory;
-import org.hl7.fhir.dstu3.model.Bundle.*;
+import org.hl7.fhir.dstu3.model.*;
+import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.dstu3.model.Bundle.BundleLinkComponent;
+import org.hl7.fhir.dstu3.model.Bundle.BundleType;
 import org.hl7.fhir.dstu3.model.ContactPoint.ContactPointSystem;
 import org.hl7.fhir.dstu3.model.DiagnosticReport.DiagnosticReportStatus;
 import org.hl7.fhir.dstu3.model.ElementDefinition.ElementDefinitionBindingComponent;
@@ -52,27 +37,27 @@ import org.junit.*;
 import org.mockito.ArgumentCaptor;
 import org.xmlunit.builder.DiffBuilder;
 import org.xmlunit.builder.Input;
-import org.xmlunit.diff.*;
+import org.xmlunit.diff.ComparisonControllers;
+import org.xmlunit.diff.DefaultNodeMatcher;
+import org.xmlunit.diff.Diff;
+import org.xmlunit.diff.ElementSelectors;
 
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.*;
 
-import ca.uhn.fhir.context.FhirContext;
+import static org.hamcrest.Matchers.contains;
-import ca.uhn.fhir.model.api.annotation.Child;
+import static org.hamcrest.Matchers.*;
-import ca.uhn.fhir.model.api.annotation.ResourceDef;
+import static org.hamcrest.Matchers.startsWith;
-import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
+import static org.junit.Assert.*;
-import ca.uhn.fhir.parser.FooMessageHeaderWithExplicitField.FooMessageSourceComponent;
+import static org.mockito.Matchers.any;
-import ca.uhn.fhir.parser.IParserErrorHandler.IParseLocation;
+import static org.mockito.Mockito.*;
-import ca.uhn.fhir.parser.PatientWithCustomCompositeExtension.FooParentExtension;
-import ca.uhn.fhir.rest.client.api.IGenericClient;
-import ca.uhn.fhir.util.TestUtil;
-import ca.uhn.fhir.validation.IValidationContext;
-import ca.uhn.fhir.validation.SingleValidationMessage;
-import ca.uhn.fhir.validation.ValidationContext;
-import ca.uhn.fhir.validation.ValidationResult;
 
 public class XmlParserDstu3Test {
-	private static FhirContext ourCtx = FhirContext.forDstu3();
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(XmlParserDstu3Test.class);
+	private static FhirContext ourCtx = FhirContext.forDstu3();
 
 	@After
 	public void after() {
@@ -83,21 +68,24 @@ public class XmlParserDstu3Test {
 	}
 
 	@Test
-	public void testEncodeBinaryWithSecurityContext() {
+	public void testBaseUrlFooResourceCorrectlySerializedInExtensionValueReference() {
-		Binary bin = new Binary();
+		String refVal = "http://my.org/FooBar";
-		bin.setContentType("text/plain");
-		bin.setContent("Now is the time".getBytes());
-		Reference securityContext = new Reference();
-		securityContext.setReference("DiagnosticReport/1");
-		bin.setSecurityContext(securityContext);
-		String encoded = ourCtx.newXmlParser().encodeResourceToString(bin);
-		ourLog.info(encoded);
-		assertThat(encoded, containsString("Binary"));
-		assertThat(encoded, containsString("<contentType value=\"text/plain\"/>"));
-		assertThat(encoded, containsString("<securityContext><reference value=\"DiagnosticReport/1\"/></securityContext>"));
-		assertThat(encoded, containsString("<content value=\"Tm93IGlzIHRoZSB0aW1l\"/>"));
-	}
 
+		Patient fhirPat = new Patient();
+		fhirPat.addExtension().setUrl("x1").setValue(new Reference(refVal));
+
+		IParser parser = ourCtx.newXmlParser();
+
+		String output = parser.encodeResourceToString(fhirPat);
+		System.out.println("output: " + output);
+
+		// Deserialize then check that valueReference value is still correct
+		fhirPat = parser.parseResource(Patient.class, output);
+
+		List<Extension> extlst = fhirPat.getExtensionsByUrl("x1");
+		Assert.assertEquals(1, extlst.size());
+		Assert.assertEquals(refVal, ((Reference) extlst.get(0).getValue()).getReference());
+	}
 
 	/**
 	 * See #544
@@ -977,6 +965,22 @@ public class XmlParserDstu3Test {
 		assertEquals("<Binary xmlns=\"http://hl7.org/fhir\"><content value=\"AQIDBA==\"/></Binary>", output);
 	}
 
+	@Test
+	public void testEncodeBinaryWithSecurityContext() {
+		Binary bin = new Binary();
+		bin.setContentType("text/plain");
+		bin.setContent("Now is the time".getBytes());
+		Reference securityContext = new Reference();
+		securityContext.setReference("DiagnosticReport/1");
+		bin.setSecurityContext(securityContext);
+		String encoded = ourCtx.newXmlParser().encodeResourceToString(bin);
+		ourLog.info(encoded);
+		assertThat(encoded, containsString("Binary"));
+		assertThat(encoded, containsString("<contentType value=\"text/plain\"/>"));
+		assertThat(encoded, containsString("<securityContext><reference value=\"DiagnosticReport/1\"/></securityContext>"));
+		assertThat(encoded, containsString("<content value=\"Tm93IGlzIHRoZSB0aW1l\"/>"));
+	}
+
 	@Test
 	public void testEncodeBundleWithContained() {
 		DiagnosticReport rpt = new DiagnosticReport();
@@ -1872,7 +1876,7 @@ public class XmlParserDstu3Test {
 
 		{
 			IParser p = ourCtx.newXmlParser();
-			p.setEncodeElements(new HashSet<String>(Arrays.asList("Patient.name", "Bundle.entry")));
+			p.setEncodeElements(new HashSet<>(Arrays.asList("Patient.name", "Bundle.entry")));
 			p.setPrettyPrint(true);
 			String out = p.encodeResourceToString(bundle);
 			ourLog.info(out);
@@ -1908,6 +1912,31 @@ public class XmlParserDstu3Test {
 
 	}
 
+	@Test
+	public void testEncodeWithEncodeElementsAppliesToChildResourcesOnly() throws Exception {
+		Patient patient = new Patient();
+		patient.getMeta().addProfile("http://profile");
+		patient.addName().setFamily("FAMILY");
+		patient.addAddress().addLine("LINE1");
+
+		Bundle bundle = new Bundle();
+		bundle.setTotal(100);
+		bundle.addEntry().setResource(patient);
+
+		{
+			IParser p = ourCtx.newXmlParser();
+			p.setEncodeElements(new HashSet<>(Arrays.asList("Patient.name")));
+			p.setEncodeElementsAppliesToChildResourcesOnly(true);
+			p.setPrettyPrint(true);
+			String out = p.encodeResourceToString(bundle);
+			ourLog.info(out);
+			assertThat(out, containsString("total"));
+			assertThat(out, containsString("Patient"));
+			assertThat(out, containsString("name"));
+			assertThat(out, not(containsString("address")));
+		}
+	}
+
 	@Test
 	public void testEncodeWithNarrative() {
 		Patient p = new Patient();
@@ -2835,7 +2864,7 @@ public class XmlParserDstu3Test {
 
 	/**
 	 * See #426
-	 * 
+	 * <p>
 	 * Value type of FOO isn't a valid datatype
 	 */
 	@Test
@@ -2934,6 +2963,30 @@ public class XmlParserDstu3Test {
 		assertEquals("value", capt.getValue());
 	}
 
+	@Test
+	public void testParseMetaUpdatedDate() {
+
+		String input = "<Bundle xmlns=\"http://hl7.org/fhir\">\n" +
+			"   <id value=\"e2ee823b-ee4d-472d-b79d-495c23f16b99\"/>\n" +
+			"   <meta>\n" +
+			"      <lastUpdated value=\"2015-06-22T15:48:57.554-04:00\"/>\n" +
+			"   </meta>\n" +
+			"   <type value=\"searchset\"/>\n" +
+			"   <base value=\"http://localhost:58109/fhir/context\"/>\n" +
+			"   <total value=\"0\"/>\n" +
+			"   <link>\n" +
+			"      <relation value=\"self\"/>\n" +
+			"      <url value=\"http://localhost:58109/fhir/context/Patient?_pretty=true\"/>\n" +
+			"   </link>\n" +
+			"</Bundle>";
+
+		Bundle b = ourCtx.newXmlParser().parseResource(Bundle.class, input);
+
+		InstantType updated = b.getMeta().getLastUpdatedElement();
+		assertEquals("2015-06-22T15:48:57.554-04:00", updated.getValueAsString());
+
+	}
+
 	@Test
 	public void testParseMetadata() throws Exception {
 
@@ -2987,43 +3040,6 @@ public class XmlParserDstu3Test {
 
 	}
 
-	public static void compareXml(String content, String reEncoded) {
-		Diff d = DiffBuilder.compare(Input.fromString(content))
-				.withTest(Input.fromString(reEncoded))
-				.withNodeMatcher(new DefaultNodeMatcher(ElementSelectors.byNameAndText))
-				.checkForSimilar()
-				.ignoreWhitespace() // this is working with newest Saxon 9.8.0-2 (not worked with 9.7.0-15
-				.ignoreComments() // this is not working even with newest Saxon 9.8.0-2
-				.withComparisonController(ComparisonControllers.Default)
-				.build();
-
-		assertTrue(d.toString(), !d.hasDifferences());
-	}
-
-	@Test
-	public void testParseMetaUpdatedDate() {
-
-		String input = "<Bundle xmlns=\"http://hl7.org/fhir\">\n" +
-				"   <id value=\"e2ee823b-ee4d-472d-b79d-495c23f16b99\"/>\n" +
-				"   <meta>\n" +
-				"      <lastUpdated value=\"2015-06-22T15:48:57.554-04:00\"/>\n" +
-				"   </meta>\n" +
-				"   <type value=\"searchset\"/>\n" +
-				"   <base value=\"http://localhost:58109/fhir/context\"/>\n" +
-				"   <total value=\"0\"/>\n" +
-				"   <link>\n" +
-				"      <relation value=\"self\"/>\n" +
-				"      <url value=\"http://localhost:58109/fhir/context/Patient?_pretty=true\"/>\n" +
-				"   </link>\n" +
-				"</Bundle>";
-
-		Bundle b = ourCtx.newXmlParser().parseResource(Bundle.class, input);
-
-		InstantType updated = b.getMeta().getLastUpdatedElement();
-		assertEquals("2015-06-22T15:48:57.554-04:00", updated.getValueAsString());
-
-	}
-
 	// TODO: this should work
 	@Test
 	@Ignore
@@ -3094,40 +3110,6 @@ public class XmlParserDstu3Test {
 		assertEquals("Patient", reincarnatedPatient.getIdElement().getResourceType());
 	}
 
-	/**
-	 * See #344
-	 */
-	@Test
-	public void testParserIsCaseSensitive() {
-		Observation obs = new Observation();
-		SampledData data = new SampledData();
-		data.setData("1 2 3");
-		data.setOrigin((SimpleQuantity) new SimpleQuantity().setValue(0L));
-		data.setPeriod(1000L);
-		obs.setValue(data);
-
-		IParser p = ourCtx.newXmlParser().setPrettyPrint(true).setParserErrorHandler(new StrictErrorHandler());
-		String encoded = p.encodeResourceToString(obs);
-		ourLog.info(encoded);
-
-		p.parseResource(encoded);
-
-		try {
-			p.parseResource(encoded.replace("Observation", "observation"));
-			fail();
-		} catch (DataFormatException e) {
-			assertEquals("DataFormatException at [[row,col {unknown-source}]: [1,1]]: Unknown resource type 'observation': Resource names are case sensitive, found similar name: 'Observation'",
-					e.getMessage());
-		}
-
-		try {
-			p.parseResource(encoded.replace("valueSampledData", "valueSampleddata"));
-			fail();
-		} catch (DataFormatException e) {
-			assertEquals("DataFormatException at [[row,col {unknown-source}]: [2,4]]: Unknown element 'valueSampleddata' found during parse", e.getMessage());
-		}
-	}
-
 	@Test(expected = DataFormatException.class)
 	public void testParseWithInvalidLocalRef() throws IOException {
 		String string = IOUtils.toString(getClass().getResourceAsStream("/bundle_with_invalid_contained_ref.xml"), StandardCharsets.UTF_8);
@@ -3226,6 +3208,40 @@ public class XmlParserDstu3Test {
 
 	}
 
+	/**
+	 * See #344
+	 */
+	@Test
+	public void testParserIsCaseSensitive() {
+		Observation obs = new Observation();
+		SampledData data = new SampledData();
+		data.setData("1 2 3");
+		data.setOrigin((SimpleQuantity) new SimpleQuantity().setValue(0L));
+		data.setPeriod(1000L);
+		obs.setValue(data);
+
+		IParser p = ourCtx.newXmlParser().setPrettyPrint(true).setParserErrorHandler(new StrictErrorHandler());
+		String encoded = p.encodeResourceToString(obs);
+		ourLog.info(encoded);
+
+		p.parseResource(encoded);
+
+		try {
+			p.parseResource(encoded.replace("Observation", "observation"));
+			fail();
+		} catch (DataFormatException e) {
+			assertEquals("DataFormatException at [[row,col {unknown-source}]: [1,1]]: Unknown resource type 'observation': Resource names are case sensitive, found similar name: 'Observation'",
+				e.getMessage());
+		}
+
+		try {
+			p.parseResource(encoded.replace("valueSampledData", "valueSampleddata"));
+			fail();
+		} catch (DataFormatException e) {
+			assertEquals("DataFormatException at [[row,col {unknown-source}]: [2,4]]: Unknown element 'valueSampleddata' found during parse", e.getMessage());
+		}
+	}
+
 	/**
 	 * See #551
 	 */
@@ -3245,7 +3261,7 @@ public class XmlParserDstu3Test {
 
 	/**
 	 * See #339
-	 * 
+	 * <p>
 	 * https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing
 	 */
 	@Test
@@ -3275,35 +3291,22 @@ public class XmlParserDstu3Test {
 
 	}
 
-	@Test
-	public void testBaseUrlFooResourceCorrectlySerializedInExtensionValueReference() {
-		String refVal = "http://my.org/FooBar";
-
-		Patient fhirPat = new Patient();
-		fhirPat.addExtension().setUrl("x1").setValue(new Reference(refVal));
-
-		IParser parser = ourCtx.newXmlParser();
-
-		String output = parser.encodeResourceToString(fhirPat);
-		System.out.println("output: " + output);
-
-		// Deserialize then check that valueReference value is still correct
-		fhirPat = parser.parseResource(Patient.class, output);
-
-		List<Extension> extlst = fhirPat.getExtensionsByUrl("x1");
-		Assert.assertEquals(1, extlst.size());
-		Assert.assertEquals(refVal, ((Reference) extlst.get(0).getValue()).getReference());
-	}
-
 	@AfterClass
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();
 	}
 
-	public static void main(String[] args) {
+	public static void compareXml(String content, String reEncoded) {
-		IGenericClient c = ourCtx.newRestfulGenericClient("http://fhir-dev.healthintersections.com.au/open");
+		Diff d = DiffBuilder.compare(Input.fromString(content))
-		// c.registerInterceptor(new LoggingInterceptor(true));
+			.withTest(Input.fromString(reEncoded))
-		c.read().resource("Patient").withId("324").execute();
+			.withNodeMatcher(new DefaultNodeMatcher(ElementSelectors.byNameAndText))
+			.checkForSimilar()
+			.ignoreWhitespace() // this is working with newest Saxon 9.8.0-2 (not worked with 9.7.0-15
+			.ignoreComments() // this is not working even with newest Saxon 9.8.0-2
+			.withComparisonController(ComparisonControllers.Default)
+			.build();
+
+		assertTrue(d.toString(), !d.hasDifferences());
 	}
 
 	@ResourceDef(name = "Patient")

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/SearchR4Test.java

@@ -1,32 +1,5 @@
 package ca.uhn.fhir.rest.server;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.hl7.fhir.instance.model.api.IBaseResource;
-import org.hl7.fhir.r4.model.*;
-import org.junit.*;
-
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.annotation.RequiredParam;
 import ca.uhn.fhir.rest.annotation.Search;
@@ -38,15 +11,46 @@ import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
 import ca.uhn.fhir.rest.gclient.StringClientParam;
 import ca.uhn.fhir.rest.param.TokenAndListParam;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
-import ca.uhn.fhir.util.*;
+import ca.uhn.fhir.util.PortUtil;
+import ca.uhn.fhir.util.TestUtil;
+import ca.uhn.fhir.util.UrlUtil;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.HumanName;
+import org.hl7.fhir.r4.model.OperationOutcome;
+import org.hl7.fhir.r4.model.Patient;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.*;
 
 public class SearchR4Test {
 
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchR4Test.class);
 	private static CloseableHttpClient ourClient;
 	private static FhirContext ourCtx = FhirContext.forR4();
 	private static TokenAndListParam ourIdentifiers;
 	private static String ourLastMethod;
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchR4Test.class);
 	private static int ourPort;
 
 	private static Server ourServer;
@@ -57,74 +61,63 @@ public class SearchR4Test {
 		ourIdentifiers = null;
 	}
 
-	@Test
+	private Bundle executeAndReturnLinkNext(HttpGet httpGet, EncodingEnum theExpectEncoding) throws IOException, ClientProtocolException {
-	public void testSearchNormal() throws Exception {
-		HttpGet httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar");
 		CloseableHttpResponse status = ourClient.execute(httpGet);
+		Bundle bundle;
 		try {
 			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(responseContent);
 			assertEquals(200, status.getStatusLine().getStatusCode());
-
+			EncodingEnum ct = EncodingEnum.forContentType(status.getEntity().getContentType().getValue().replaceAll(";.*", "").trim());
-			assertEquals("search", ourLastMethod);
+			assertEquals(theExpectEncoding, ct);
-
+			bundle = ct.newParser(ourCtx).parseResource(Bundle.class, responseContent);
-			assertEquals("foo", ourIdentifiers.getValuesAsQueryTokens().get(0).getValuesAsQueryTokens().get(0).getSystem());
+			assertEquals(10, bundle.getEntry().size());
-			assertEquals("bar", ourIdentifiers.getValuesAsQueryTokens().get(0).getValuesAsQueryTokens().get(0).getValue());
+			String linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
+			assertNotNull(linkNext);
 		} finally {
 			IOUtils.closeQuietly(status.getEntity().getContent());
 		}
-
+		return bundle;
 	}
 
 	@Test
-	public void testSearchWithInvalidChain() throws Exception {
+	public void testPagingPreservesElements() throws Exception {
-		HttpGet httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier.chain=foo%7Cbar");
-		CloseableHttpResponse status = ourClient.execute(httpGet);
-		try {
-			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
-			ourLog.info(responseContent);
-			assertEquals(400, status.getStatusLine().getStatusCode());
-
-			OperationOutcome oo = (OperationOutcome) ourCtx.newJsonParser().parseResource(responseContent);
-			assertEquals(
-					"Invalid search parameter \"identifier.chain\". Parameter contains a chain (.chain) and chains are not supported for this parameter (chaining is only allowed on reference parameters)",
-					oo.getIssueFirstRep().getDiagnostics());
-		} finally {
-			IOUtils.closeQuietly(status.getEntity().getContent());
-		}
-
-	}
-
-	
-	@Test
-	public void testPagingPreservesEncodingJson() throws Exception {
 		HttpGet httpGet;
 		String linkNext;
 		Bundle bundle;
+		String linkSelf;
 
 		// Initial search
-		httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar&_format=json");
+		httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar&_elements=name");
 		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
+		assertThat(toJson(bundle), not(containsString("active")));
+		linkSelf = bundle.getLink(Constants.LINK_SELF).getUrl();
+		assertThat(linkSelf, containsString("_elements=name"));
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=json"));
+		assertThat(linkNext, containsString("_elements=name"));
+
+		ourLog.info(toJson(bundle));
 
 		// Fetch the next page
 		httpGet = new HttpGet(linkNext);
 		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
+		assertThat(toJson(bundle), not(containsString("active")));
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=json"));
+		assertThat(linkNext, containsString("_elements=name"));
 
 		// Fetch the next page
 		httpGet = new HttpGet(linkNext);
 		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
+		assertThat(toJson(bundle), not(containsString("active")));
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=json"));
+		assertThat(linkNext, containsString("_elements=name"));
 
 		// Fetch the next page
 		httpGet = new HttpGet(linkNext);
 		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
+		assertThat(toJson(bundle), not(containsString("active")));
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=json"));
+		assertThat(linkNext, containsString("_elements=name"));
 
 	}
 
@@ -161,34 +154,35 @@ public class SearchR4Test {
 	}
 
 	@Test
-	public void testPagingPreservesEncodingXml() throws Exception {
+	public void testPagingPreservesEncodingJson() throws Exception {
 		HttpGet httpGet;
 		String linkNext;
 		Bundle bundle;
 
 		// Initial search
-		httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar&_format=xml");
+		httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar&_format=json");
-		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
+		assertThat(toJson(bundle), containsString("active"));
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=xml"));
+		assertThat(linkNext, containsString("_format=json"));
 
 		// Fetch the next page
 		httpGet = new HttpGet(linkNext);
-		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=xml"));
+		assertThat(linkNext, containsString("_format=json"));
 
 		// Fetch the next page
 		httpGet = new HttpGet(linkNext);
-		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=xml"));
+		assertThat(linkNext, containsString("_format=json"));
 
 		// Fetch the next page
 		httpGet = new HttpGet(linkNext);
-		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.JSON);
 		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
-		assertThat(linkNext, containsString("_format=xml"));
+		assertThat(linkNext, containsString("_format=json"));
 
 	}
 
@@ -260,25 +254,75 @@ public class SearchR4Test {
 
 	}
 
-	private Bundle executeAndReturnLinkNext(HttpGet httpGet, EncodingEnum theExpectEncoding) throws IOException, ClientProtocolException {
+	@Test
-		CloseableHttpResponse status = ourClient.execute(httpGet);
+	public void testPagingPreservesEncodingXml() throws Exception {
+		HttpGet httpGet;
+		String linkNext;
 		Bundle bundle;
+
+		// Initial search
+		httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar&_format=xml");
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
+		assertThat(linkNext, containsString("_format=xml"));
+
+		// Fetch the next page
+		httpGet = new HttpGet(linkNext);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
+		assertThat(linkNext, containsString("_format=xml"));
+
+		// Fetch the next page
+		httpGet = new HttpGet(linkNext);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
+		assertThat(linkNext, containsString("_format=xml"));
+
+		// Fetch the next page
+		httpGet = new HttpGet(linkNext);
+		bundle = executeAndReturnLinkNext(httpGet, EncodingEnum.XML);
+		linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
+		assertThat(linkNext, containsString("_format=xml"));
+
+	}
+
+	@Test
+	public void testSearchNormal() throws Exception {
+		HttpGet httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier=foo%7Cbar");
+		CloseableHttpResponse status = ourClient.execute(httpGet);
 		try {
 			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
 			ourLog.info(responseContent);
 			assertEquals(200, status.getStatusLine().getStatusCode());
-			EncodingEnum ct = EncodingEnum.forContentType(status.getEntity().getContentType().getValue().replaceAll(";.*", "").trim());
+
-			assertEquals(theExpectEncoding, ct);
+			assertEquals("search", ourLastMethod);
-			bundle = ct.newParser(ourCtx).parseResource(Bundle.class, responseContent);
+
-			assertEquals(10, bundle.getEntry().size());
+			assertEquals("foo", ourIdentifiers.getValuesAsQueryTokens().get(0).getValuesAsQueryTokens().get(0).getSystem());
-			String linkNext = bundle.getLink(Constants.LINK_NEXT).getUrl();
+			assertEquals("bar", ourIdentifiers.getValuesAsQueryTokens().get(0).getValuesAsQueryTokens().get(0).getValue());
-			assertNotNull(linkNext);
 		} finally {
 			IOUtils.closeQuietly(status.getEntity().getContent());
 		}
-		return bundle;
+
 	}
 
+	@Test
+	public void testSearchWithInvalidChain() throws Exception {
+		HttpGet httpGet = new HttpGet("http://localhost:" + ourPort + "/Patient?identifier.chain=foo%7Cbar");
+		CloseableHttpResponse status = ourClient.execute(httpGet);
+		try {
+			String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8);
+			ourLog.info(responseContent);
+			assertEquals(400, status.getStatusLine().getStatusCode());
+
+			OperationOutcome oo = (OperationOutcome) ourCtx.newJsonParser().parseResource(responseContent);
+			assertEquals(
+				"Invalid search parameter \"identifier.chain\". Parameter contains a chain (.chain) and chains are not supported for this parameter (chaining is only allowed on reference parameters)",
+				oo.getIssueFirstRep().getDiagnostics());
+		} finally {
+			IOUtils.closeQuietly(status.getEntity().getContent());
+		}
+
+	}
 
 	@Test
 	public void testSearchWithPostAndInvalidParameters() throws Exception {
@@ -308,6 +352,10 @@ public class SearchR4Test {
 
 	}
 
+	private String toJson(Bundle theBundle) {
+		return ourCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(theBundle);
+	}
+
 	@AfterClass
 	public static void afterClassClearContext() throws Exception {
 		ourServer.stop();
@@ -357,8 +405,9 @@ public class SearchR4Test {
 			for (int i = 0; i < 200; i++) {
 				Patient patient = new Patient();
 				patient.addName(new HumanName().setFamily("FAMILY"));
+				patient.setActive(true);
 				patient.getIdElement().setValue("Patient/" + i);
-				retVal.add((Patient) patient);
+				retVal.add(patient);
 			}
 			return retVal;
 		}

src/changes/changes.xml

@@ -175,11 +175,26 @@
 				was not encoded correctly. Thanks to Malcolm McRoberts for the pull
 				request with fix and test case!
 			</action>
+			<action type="add">
+				Bundle resources did not have their version encoded when serializing
+				in FHIR resource (XML/JSON) format.
+			</action>
 			<action type="add">
 				The Binary resource endpoint now supports the `X-Security-Context` header when
 				reading or writing Binary contents using their native Content-Type (i.e exchanging
 				the raw binary with the server, as opposed to exchanging a FHIR resource).
 			</action>
+			<action type="fix">
+				When paging through multiple pages of search results, if the 
+				client had requested a subset of resources to be returned using the
+				<![CDATA[<code>_elements</code>]]> parameter, the elements list
+				was lost after the first page of results.
+				In addition, elements will not remove elements from
+				search/history Bundles (i.e. elements from the Bundle itself, as opposed
+				to elements in the entry resources) unless the Bundle elements are
+				explicitly listed, e.g. <![CDATA[<code>_include=Bundle.total</code>]]>.
+				Thanks to @parisni for reporting!
+			</action>
 		</release>
 		<release version="3.0.0" date="2017-09-27">
 			<action type="add">