Fix #454 - Chinese characters were not being correctly normalized

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java

@@ -1584,11 +1584,23 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
 
 	public static String normalizeString(String theString) {
 		char[] out = new char[theString.length()];
-		theString = Normalizer.normalize(theString, Normalizer.Form.NFD);
+
+		/*
+		 * The following block of code is used to strip out diacritical marks from latin script
+		 * and also convert to upper case. E.g. "jåmes" becomes "JAMES".
+		 * 
+		 * See http://www.unicode.org/charts/PDF/U0300.pdf for the logic
+		 * behind stripping 0300-036F
+		 * 
+		 * See #454 for an issue where we were completely stripping non latin characters
+		 */
+		String string = Normalizer.normalize(theString, Normalizer.Form.NFD);
 		int j = 0;
-		for (int i = 0, n = theString.length(); i < n; ++i) {
-			char c = theString.charAt(i);
-			if (c <= '\u007F') {
+		for (int i = 0, n = string.length(); i < n; ++i) {
+			char c = string.charAt(i);
+			if (c >= '\u0300' && c <= '\u036F') {
+				continue;
+			} else {
 				out[j++] = c;
 			}
 		}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDaoTest.java

@@ -10,12 +10,10 @@ import ca.uhn.fhir.model.dstu2.composite.PeriodDt;
 import ca.uhn.fhir.model.dstu2.resource.Condition;
 import ca.uhn.fhir.model.dstu2.resource.Observation;
 import ca.uhn.fhir.model.primitive.DateTimeDt;
-import ca.uhn.fhir.rest.annotation.ResourceParam;
 import ca.uhn.fhir.rest.param.ReferenceParam;
-import ca.uhn.fhir.rest.param.ResourceParameter;
 import ca.uhn.fhir.util.TestUtil;
 
-public class BaseFhirDaoTest  extends BaseJpaTest {
+public class BaseHapiFhirDaoTest  extends BaseJpaTest {
 
 	private static FhirContext ourCtx = FhirContext.forDstu2();
 
@@ -40,6 +38,13 @@ public class BaseFhirDaoTest  extends BaseJpaTest {
 		observation.setEffective(period);
 		
 	}
+	
+	@Test
+	public void testNormalizeString() {
+		assertEquals("TEST TEST", BaseHapiFhirDao.normalizeString("TEST teSt"));
+		assertEquals("AEIØU", BaseHapiFhirDao.normalizeString("åéîøü"));
+		assertEquals("杨浩", BaseHapiFhirDao.normalizeString("杨浩"));
+	}
 
 
 	@Override

hapi-fhir-structures-dstu3/src/test/java/org/hl7/fhir/dstu3/hapi/rest/server/ServerConformanceProviderDstu3Test.java

@@ -563,6 +563,32 @@ public class ServerConformanceProviderDstu3Test {
 		assertEquals(2, param.getChain().size());
 	}
 
+	@Test
+	public void testSearchReferenceParameterWithList() throws Exception {
+
+		RestfulServer rsNoType = new RestfulServer(ourCtx);
+		rsNoType.registerProvider(new SearchProviderWithListNoType());
+		ServerConformanceProvider sc = new ServerConformanceProvider(rsNoType);
+		rsNoType.setServerConformanceProvider(sc);
+		rsNoType.init(createServletConfig());
+
+		Conformance conformance = sc.getServerConformance(createHttpServletRequest());
+		String confNoType = ourCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(conformance);
+		ourLog.info(confNoType);
+
+		RestfulServer rsWithType = new RestfulServer(ourCtx);
+		rsWithType.registerProvider(new SearchProviderWithListWithType());
+		ServerConformanceProvider scWithType = new ServerConformanceProvider(rsWithType);
+		rsWithType.setServerConformanceProvider(scWithType);
+		rsWithType.init(createServletConfig());
+
+		Conformance conformanceWithType = scWithType.getServerConformance(createHttpServletRequest());
+		String confWithType = ourCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(conformanceWithType);
+		ourLog.info(confWithType);
+
+		assertEquals(confNoType, confWithType);
+	}
+
 	@Test
 	public void testSystemHistorySupported() throws Exception {
 
@@ -849,6 +875,39 @@ public class ServerConformanceProviderDstu3Test {
 
 	}
 
+	public static class SearchProviderWithListNoType  implements IResourceProvider {
+
+		@Override
+		public Class<? extends IBaseResource> getResourceType() {
+			return Patient.class;
+		}
+
+
+
+		@Search()
+		public List<Patient> findPatient1(@Description(shortDefinition = "The organization at which this person is a patient") @RequiredParam(name = Patient.SP_ORGANIZATION) ReferenceAndListParam theIdentifier) {
+			return null;
+		}
+
+	}
+
+	public static class SearchProviderWithListWithType  implements IResourceProvider {
+
+		@Override
+		public Class<? extends IBaseResource> getResourceType() {
+			return Patient.class;
+		}
+
+
+
+		@Search(type=Patient.class)
+		public List<Patient> findPatient1(@Description(shortDefinition = "The organization at which this person is a patient") @RequiredParam(name = Patient.SP_ORGANIZATION) ReferenceAndListParam theIdentifier) {
+			return null;
+		}
+
+	}
+
+	
 	public static class SystemHistoryProvider {
 
 		@History

src/changes/changes.xml

@@ -180,6 +180,11 @@
 				JPA server shouldn't report a totalCount in Bundle of "-1" when
 				there are no results
 			</action>
+			<action type="fix" issue="454">
+				JPA server was not correctly normalizing strings with non-latin characters
+				(e.g. Chinese chars). Thanks to GitHub user @YinAqu for reporting and providing
+				some great analysis of the issue! 
+			</action>
 		</release>
 		<release version="2.0" date="2016-08-30">
 			<action type="fix">