Rework valueset index for MySQL (#2269)

* Rework valueset index for MySQL

* Add changelog

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_3_0/2269-rework-valueset-index.yaml

@@ -0,0 +1,6 @@
+---
+type: fix
+issue: 2269
+title: "A database index in the JPA server was added in HAPI FHIR 5.2.0 and Smile CDR 2020.11 that
+   exceeded the maximum index length in MySQL, preventing server upgrades on that database platform.
+   This has been corrected."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConcept.java

@@ -36,11 +36,11 @@ import static org.apache.commons.lang3.StringUtils.left;
 import static org.apache.commons.lang3.StringUtils.length;
 
 /*
- * DM 2019-08-01 - Do not use IDX_VALUESET_CONCEPT_CS_CD; this was previously used as an index so reusing the name will
+ * DM 2019-08-01 - Do not use IDX_VALUESET_CONCEPT_CS_CD or IDX_VALUESET_CONCEPT_CS_CODE; this was previously used as an index so reusing the name will
  * bork up migration tasks.
  */
 @Table(name = "TRM_VALUESET_CONCEPT", uniqueConstraints = {
-	@UniqueConstraint(name = "IDX_VS_CONCEPT_CS_CODE", columnNames = {"VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL"}),
+	@UniqueConstraint(name = "IDX_VS_CONCEPT_CSCD", columnNames = {"VALUESET_PID", "SYSTEM_URL", "CODEVAL"}),
 	@UniqueConstraint(name = "IDX_VS_CONCEPT_ORDER", columnNames = {"VALUESET_PID", "VALUESET_ORDER"})
 })
 @Entity()

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/TestUtil.java

@@ -33,7 +33,21 @@ import org.hibernate.validator.constraints.Length;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.InstantType;
 
-import javax.persistence.*;
+import javax.persistence.Column;
+import javax.persistence.Embedded;
+import javax.persistence.EmbeddedId;
+import javax.persistence.Entity;
+import javax.persistence.ForeignKey;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.Lob;
+import javax.persistence.OneToMany;
+import javax.persistence.OneToOne;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+import javax.persistence.Transient;
+import javax.persistence.UniqueConstraint;
 import javax.validation.constraints.Size;
 import java.io.IOException;
 import java.io.InputStream;
@@ -43,7 +57,9 @@ import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -99,6 +115,41 @@ public class TestUtil {
 	}
 
 	private static void scanClass(Set<String> theNames, Class<?> theClazz, boolean theIsSuperClass) {
+		Map<String, Integer> columnNameToLength = new HashMap<>();
+
+		scanClassOrSuperclass(theNames, theClazz, theIsSuperClass, columnNameToLength);
+
+		Table table = theClazz.getAnnotation(Table.class);
+		if (table != null) {
+
+			// This is the length for MySQL per https://dev.mysql.com/doc/refman/8.0/en/innodb-limits.html
+			// No idea why 3072.. what a weird limit but I'm sure they have their reason.
+			int maxIndexLength = 3072;
+
+			for (UniqueConstraint nextIndex : table.uniqueConstraints()) {
+				int indexLength = calculateIndexLength(nextIndex.columnNames(), columnNameToLength, theClazz, nextIndex.name());
+				if (indexLength > maxIndexLength) {
+					throw new IllegalStateException("Index '" + nextIndex.name() + "' is too long. Length is " + indexLength + " and must not exceed " + maxIndexLength + " which is the maximum MySQL length");
+				}
+			}
+
+		}
+
+	}
+
+	private static int calculateIndexLength(String[] theColumnNames, Map<String, Integer> theColumnNameToLength, Class<?> theClazz, String theIndexName) {
+		int retVal = 0;
+		for (String nextName : theColumnNames) {
+			Integer nextLength = theColumnNameToLength.get(nextName);
+			if (nextLength == null) {
+				throw new IllegalStateException("Index '" + theIndexName + "' references unknown column: " + nextName);
+			}
+			retVal += nextLength;
+		}
+		return retVal;
+	}
+
+	private static void scanClassOrSuperclass(Set<String> theNames, Class<?> theClazz, boolean theIsSuperClass, Map<String, Integer> columnNameToLength) {
 		ourLog.info("Scanning: {}", theClazz.getSimpleName());
 
 		Subselect subselect = theClazz.getAnnotation(Subselect.class);
@@ -134,12 +185,25 @@ public class TestUtil {
 				boolean isField = nextField.getAnnotation(org.hibernate.search.annotations.Field.class) != null;
 				Validate.isTrue(
 					hasEmbedded ||
-					hasColumn ||
+						hasColumn ||
 						hasJoinColumn ||
 						isOtherSideOfOneToManyMapping ||
 						isOtherSideOfOneToOneMapping ||
 						hasEmbeddedId ||
 						isField, "Non-transient has no @Column or @JoinColumn or @EmbeddedId: " + nextField);
+
+				if (hasColumn) {
+					String columnName = nextField.getAnnotation(Column.class).name();
+					int columnLength = nextField.getAnnotation(Column.class).length();
+					if (nextField.getType().isAssignableFrom(String.class)) {
+						columnLength = columnLength * 4;
+					} else {
+						columnLength = 16;
+					}
+
+					columnNameToLength.put(columnName, columnLength);
+				}
+
 			}
 
 
@@ -149,7 +213,7 @@ public class TestUtil {
 			return;
 		}
 
-		scanClass(theNames, theClazz.getSuperclass(), true);
+		scanClassOrSuperclass(theNames, theClazz.getSuperclass(), true, columnNameToLength);
 	}
 
 	private static void scan(AnnotatedElement theAnnotatedElement, Set<String> theNames, boolean theIsSuperClass, boolean theIsView) {

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -70,7 +70,19 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		init500(); // 20200218 - 20200513
 		init501(); // 20200514 - 20200515
 		init510(); // 20200516 - 20201028
-		init520(); // 20201029 - Present
+		init520(); // 20201029 -
+		init530();
+	}
+
+	private void init530() {
+		Builder version = forVersion(VersionEnum.V5_3_0);
+		version
+			.onTable("TRM_VALUESET_CONCEPT")
+			.dropIndex("20210104.1", "IDX_VS_CONCEPT_CS_CODE");
+		version
+			.onTable("TRM_VALUESET_CONCEPT")
+			.addIndex("20210104.2", "IDX_VS_CONCEPT_CSCD").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "CODEVAL").doNothing();
+
 	}
 
 	protected void init520() {
@@ -174,7 +186,7 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 		Builder.BuilderWithTableName trmValueSetComp = version.onTable("TRM_VALUESET_CONCEPT");
 		trmValueSetComp.addColumn("20201028.1", "SYSTEM_VER").nullable().type(ColumnTypeEnum.STRING, 200);
 		trmValueSetComp.dropIndex("20201028.2", "IDX_VS_CONCEPT_CS_CD");
-		trmValueSetComp.addIndex("20201028.3", "IDX_VS_CONCEPT_CS_CODE").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL");
+		trmValueSetComp.addIndex("20201028.3", "IDX_VS_CONCEPT_CS_CODE").unique(true).withColumns("VALUESET_PID", "SYSTEM_URL", "SYSTEM_VER", "CODEVAL").doNothing();
 	}
 
 	protected void init510_20200725() {