Rel 5 6 1 mergeback (#3228)

* 3138 externalized binary packages (#3139)

* Add test and impl

* Add changelog

* Fix test

* Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/3138-support-externalized-binaries.yaml

* add beans to test configs

* Typo

* Update hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/IBinaryStorageSvc.java

Co-authored-by: michaelabuckley <michaelabuckley@gmail.com>

* Update hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/binstore/NullBinaryStorageSvcImpl.java

Co-authored-by: michaelabuckley <michaelabuckley@gmail.com>

Co-authored-by: Kevin Dougan SmileCDR <72025369+KevinDougan-SmileCDR@users.noreply.github.com>
Co-authored-by: michaelabuckley <michaelabuckley@gmail.com>

* 3131 - Added support for the lookup operation in the Remote Terminology code (#3134)

* Remove leading underscores from identifiers (#3146)

* Version bump

* License files

* version.yaml

* Add executeRawSql() method to Migrator (#3183)

* Add implementation, add test

* Add javadoc

* Add another helper function

* Bump version

* 3153 update unknowncodesystemwarningvallidationsupport to have configurable severities (#3155)

* 3153 updating the unknowncodesystemwarningvalidationsupport class

* 3153 updating version for smile

* 3153 update broken tests

* 3153 adding log

* 3153 cleanup

* 3153 cleanup

Co-authored-by: leif stawnyczy <leifstawnyczy@leifs-MacBook-Pro.local>

* Add backport info

* Avoid creating ResourcePersistentId for placeholder resources with null ID (#3158)

* Add check before mapping storage ID to resource ID in TransactionDetails.

* Add change log.

* Changed to instead prevent creation of ResourcePersistentId with null ID value.

* Changed to instead prevent ResourcePersistentId being created with null resource ID.

Co-authored-by: ianmarshall <ian@simpatico.ai>

* Fix bug loading packages in non-database mode (#3199)

* Add implementation

* Add changelog

* Add backports

* 3164 updating code review points (#3165)

* 3164 updating code review points

* 3164 updating code review points

Co-authored-by: leif stawnyczy <leifstawnyczy@leifs-MacBook-Pro.local>

* Add executeRawSqlStub (#3203)

Co-authored-by: Joanne Mendoza <joanne@smilecdr.com>

* Add new version

* begin with failing test

* fix bug

* change log

* change log

* Add backport informationm

* begin with failing test

* fix issue

* change log

* code review

* Add backport info. Add folder

* fixed

* Add test

* Update test

* Add backport info

* Add 5.5.4

* Remove jpaconstants

Co-authored-by: Kevin Dougan SmileCDR <72025369+KevinDougan-SmileCDR@users.noreply.github.com>
Co-authored-by: michaelabuckley <michaelabuckley@gmail.com>
Co-authored-by: TipzCM <leif.stawnyczy@gmail.com>
Co-authored-by: leif stawnyczy <leifstawnyczy@leifs-MacBook-Pro.local>
Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com>
Co-authored-by: ianmarshall <ian@simpatico.ai>
Co-authored-by: Joanne Mendoza <joannemmendoza@gmail.com>
Co-authored-by: Joanne Mendoza <joanne@smilecdr.com>
Co-authored-by: Ken Stevens <khstevens@gmail.com>
Co-authored-by: katie_smilecdr <katie@smilecdr.com>

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/VersionEnum.java

@@ -78,7 +78,9 @@ public enum VersionEnum {
 	V5_5_1,
 	V5_5_2,
 	V5_5_3,
+	V5_5_4,
 	V5_6_0,
+	V5_6_1,
 	V5_7_0
 	;
 

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_1/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2021-12-03"
+codename: "Raccoon"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3153-updating-unknowncodesystem-to-allow-warnings-to-be-registered-as-warnings.yaml

@@ -1,6 +1,7 @@
 ---
 type: fix
 issue: 3153
+backport: 5.6.1
 jira: SMILE-3289
 title: "Updated UnknownCodeSystemWarningValidationSupport to allow the throwing of warnings if
 configured to do so."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3158-creation-of-versioned-placeholders.yaml

@@ -1,5 +1,6 @@
 ---
 type: fix
 issue: 3158
+backport: 5.6.1
 title: "Resource links were previously not being consistently created in cases where references were versioned and
   pointing to recently auto-created placeholder resources."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3198-package-loading-woes.yaml

@@ -2,4 +2,5 @@
 type: change
 issue: 3198
 jira: SMILE-3452
+backport: 5.6.1
 title: "Fixed a regression where packages would fail to load if HAPI-FHIR was operating in DATABASE binary storage mode."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3211-reindex-performance.yaml

@@ -1,4 +1,5 @@
 ---
 type: fix
+backport: 5.6.1
 jira: SMILE-3472
 title: "Fixed a serious performance issue with the `$reindex` operation."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3215-nextURL-does-not-work-for-a-certain-offset-with-everything.yaml

@@ -1,4 +1,5 @@
 ---
 type: fix
 issue: 3215
+backport: 5.6.1
 title: "$everything operation returns a 500 error when querying for a page when _getpagesoffset is greater than or equal to 300. This has been corrected."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_7_0/3216-mdm-possible-match-to-match.yaml

@@ -1,3 +1,4 @@
 ---
 type: fix
+backport: 5.6.1
 title: "MDM was throwing a NullPointerException when upgrading a match from POSSIBLE_MATCH to MATCH.  This has been corrected."

hapi-fhir-sql-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/api/Builder.java

@@ -46,7 +46,10 @@ import org.apache.commons.lang3.Validate;
 import org.intellij.lang.annotations.Language;
 
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -73,11 +76,21 @@ public class Builder {
 	}
 
 	public BuilderCompleteTask executeRawSql(String theVersion, @Language("SQL") String theSql) {
-		ExecuteRawSqlTask task = new ExecuteRawSqlTask(myRelease, theVersion).addSql(theSql);
+		ExecuteRawSqlTask task = executeRawSqlOptional(false, theVersion, theSql);
-		mySink.addTask(task);
 		return new BuilderCompleteTask(task);
 	}
 
+	public void executeRawSqlStub(String theVersion, @Language("SQL") String theSql) {
+		executeRawSqlOptional(true, theVersion, theSql);
+	}
+
+	private ExecuteRawSqlTask executeRawSqlOptional(boolean theDoNothing, String theVersion, @Language("SQL") String theSql) {
+		ExecuteRawSqlTask task = new ExecuteRawSqlTask(myRelease, theVersion).addSql(theSql);
+		task.setDoNothing(theDoNothing);
+		mySink.addTask(task);
+		return task;
+	}
+
 	public Builder initializeSchema(String theVersion, ISchemaInitializationProvider theSchemaInitializationProvider) {
 		mySink.addTask(new InitializeSchemaTask(myRelease, theVersion, theSchemaInitializationProvider));
 		return this;
@@ -96,6 +109,41 @@ public class Builder {
 		return this;
 	}
 
+	/**
+	 * Builder method to define a raw SQL execution migration that needs to take place against multiple database types,
+	 * and the SQL they need to use is not equal. Provide a map of driver types to SQL statements.
+	 *
+	 * @param theVersion The version of the migration.
+	 * @param theDriverToSql Map of driver types to SQL statements.
+	 * @return
+	 */
+	public Builder executeRawSql(String theVersion, Map<DriverTypeEnum, String> theDriverToSql) {
+		Map<DriverTypeEnum, List<String>> singleSqlStatementMap = new HashMap<>();
+		theDriverToSql.entrySet().stream()
+			.forEach(entry -> {
+					singleSqlStatementMap.put(entry.getKey(), Collections.singletonList(entry.getValue()));
+				});
+		return executeRawSqls(theVersion, singleSqlStatementMap);
+	}
+
+	/**
+	 * Builder method to define a raw SQL execution migration that needs to take place against multiple database types,
+	 * and the SQL they need to use is not equal, and there are multiple sql commands for a given database.
+	 * Provide a map of driver types to list of SQL statements.
+	 *
+	 * @param theVersion The version of the migration.
+	 * @param theDriverToSqls Map of driver types to list of SQL statements.
+	 * @return
+	 */
+	public Builder executeRawSqls(String theVersion, Map<DriverTypeEnum, List<String>> theDriverToSqls) {
+		ExecuteRawSqlTask executeRawSqlTask = new ExecuteRawSqlTask(myRelease, theVersion);
+		theDriverToSqls.entrySet().stream()
+			.forEach(entry -> {
+				entry.getValue().forEach(sql -> executeRawSqlTask.addSql(entry.getKey(), sql));
+			});
+		mySink.addTask(executeRawSqlTask);
+		return this;
+	}
 
 	// Flyway doesn't support these kinds of migrations
 	@Deprecated

hapi-fhir-sql-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/ExecuteRawSqlTaskTest.java

@@ -6,6 +6,7 @@ import ca.uhn.fhir.util.VersionEnum;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -81,5 +82,57 @@ public class ExecuteRawSqlTaskTest extends BaseTest {
 		}
 	}
 
+	@ParameterizedTest(name = "{index}: {0}")
+	@MethodSource("data")
+	public void testDriverTypeBasedRawSqlExecution(Supplier<TestDatabaseDetails> theTestDatabaseDetails) {
+		//Given
+		before(theTestDatabaseDetails);
+		executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255))");
 
+		BaseMigrationTasks<VersionEnum> tasks = new BaseMigrationTasks<>();
+		Map<DriverTypeEnum, String> driverToSql = new HashMap<>();
+
+		//When
+		driverToSql.put(DriverTypeEnum.H2_EMBEDDED, "INSERT INTO SOMETABLE (PID, TEXTCOL) VALUES (123, 'abc')");
+		driverToSql.put(DriverTypeEnum.DERBY_EMBEDDED, "INSERT INTO SOMETABLE (PID, TEXTCOL) VALUES (456, 'def')");
+		tasks
+			.forVersion(VersionEnum.V4_0_0)
+			.executeRawSql("2001.01", driverToSql);
+
+		getMigrator().addTasks(tasks.getTasks(VersionEnum.V0_1, VersionEnum.V4_0_0));
+		getMigrator().migrate();
+
+		List<Map<String, Object>> output = executeQuery("SELECT PID,TEXTCOL FROM SOMETABLE");
+		//Then
+		if (theTestDatabaseDetails.get().getDriverType() == DriverTypeEnum.H2_EMBEDDED) {
+			assertEquals(1, output.size());
+			assertEquals(123L, output.get(0).get("PID"));
+			assertEquals("abc", output.get(0).get("TEXTCOL"));
+		} else if (theTestDatabaseDetails.get().getDriverType() == DriverTypeEnum.DERBY_EMBEDDED) {
+			assertEquals(1, output.size());
+			assertEquals(456L, output.get(0).get("PID"));
+			assertEquals("def", output.get(0).get("TEXTCOL"));
+		} else {
+			assertEquals(0, output.size());
+		}
+	}
+
+	@ParameterizedTest(name = "{index}: {0}")
+	@MethodSource("data")
+	public void testExecuteRawSqlStub(Supplier<TestDatabaseDetails> theTestDatabaseDetails) {
+		//Given
+		before(theTestDatabaseDetails);
+		executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255))");
+
+		BaseMigrationTasks<VersionEnum> tasks = new BaseMigrationTasks<>();
+		tasks.forVersion(VersionEnum.V4_0_0)
+			.executeRawSqlStub("2001.01", "INSERT INTO SOMETABLE (PID, TEXTCOL) VALUES (123, 'abc')");
+
+		getMigrator().addTasks(tasks.getTasks(VersionEnum.V0_1, VersionEnum.V4_0_0));
+		getMigrator().migrate();
+
+		List<Map<String, Object>> output = executeQuery("SELECT PID,TEXTCOL FROM SOMETABLE");
+
+		assertEquals(0, output.size());
+	}
 }