fixed batch jobs on Postgres (#5022)

* fixed issue

* fixed pipeline issue

* review changes

---------

Co-authored-by: Steven Li <steven@smilecdr.com>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_8_0/5021-unable-to-run-batch-jobs-on-postgres.yaml

@@ -0,0 +1,4 @@
+---
+type: fix
+issue: 5021
+title: "Previously, running batch jobs with postgresql as the db will result in SQL error. This has now been fixed."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2JobInstanceEntity.java

@@ -42,6 +42,7 @@ import java.util.Date;
 
 import static ca.uhn.fhir.batch2.model.JobDefinition.ID_MAX_LENGTH;
 import static ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity.ERROR_MSG_MAX_LENGTH;
+import static ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity.WARNING_MSG_MAX_LENGTH;
 import static org.apache.commons.lang3.StringUtils.left;
 
 @Entity
@@ -113,8 +114,7 @@ public class Batch2JobInstanceEntity implements Serializable {
 	private String myEstimatedTimeRemaining;
 	@Column(name = "CUR_GATED_STEP_ID", length = ID_MAX_LENGTH, nullable = true)
 	private String myCurrentGatedStepId;
-	@Lob
+	@Column(name = "WARNING_MSG", length = WARNING_MSG_MAX_LENGTH, nullable = true)
-	@Column(name = "WARNING_MSG", nullable = true)
 	private String myWarningMessages;
 
 	/**

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java

@@ -53,6 +53,7 @@ import static org.apache.commons.lang3.StringUtils.left;
 public class Batch2WorkChunkEntity implements Serializable {
 
 	public static final int ERROR_MSG_MAX_LENGTH = 500;
+	public static final int WARNING_MSG_MAX_LENGTH = 4000;
 	private static final long serialVersionUID = -6202771941965780558L;
 	@Id
 	@Column(name = "ID", length = ID_MAX_LENGTH)
@@ -96,8 +97,7 @@ public class Batch2WorkChunkEntity implements Serializable {
 	private String myErrorMessage;
 	@Column(name = "ERROR_COUNT", nullable = false)
 	private int myErrorCount;
-	@Lob
+	@Column(name = "WARNING_MSG", length = WARNING_MSG_MAX_LENGTH, nullable = true)
-	@Column(name = "WARNING_MSG", nullable = true)
 	private String myWarningMessage;
 
 	/**

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java

@@ -162,13 +162,15 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 			.onTable("BT2_WORK_CHUNK")
 			.addColumn("20230524.1", "WARNING_MSG")
 			.nullable()
-			.type(ColumnTypeEnum.CLOB);
+			.type(ColumnTypeEnum.CLOB)
+			.doNothing(); // the migration below is the better implementation
 
 		version
 			.onTable("BT2_JOB_INSTANCE")
 			.addColumn("20230524.2", "WARNING_MSG")
 			.nullable()
-			.type(ColumnTypeEnum.CLOB);
+			.type(ColumnTypeEnum.CLOB)
+			.doNothing(); // the migration below is the better implementation
 
 		// adding indexes to foreign keys
 		// this makes our table scans more efficient,
@@ -277,6 +279,29 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks<VersionEnum> {
 			.unique(false)
 			.withColumns("CONCEPT_MAP_GRP_ELM_PID")
 			.onlyAppliesToPlatforms(NON_AUTOMATIC_FK_INDEX_PLATFORMS);
+
+		// add warning message to batch job instance using limited varchar column to store
+		version
+			.onTable("BT2_WORK_CHUNK")
+			.dropColumn("20230622.1", "WARNING_MSG")
+			.failureAllowed();
+
+		version
+			.onTable("BT2_WORK_CHUNK")
+			.addColumn("20230622.2", "WARNING_MSG")
+			.nullable()
+			.type(ColumnTypeEnum.STRING, 4000);
+
+		version
+			.onTable("BT2_JOB_INSTANCE")
+			.dropColumn("20230622.3", "WARNING_MSG")
+			.failureAllowed();
+
+		version
+			.onTable("BT2_JOB_INSTANCE")
+			.addColumn("20230622.4", "WARNING_MSG")
+			.nullable()
+			.type(ColumnTypeEnum.STRING, 4000);
 	}
 
 	protected void init660() {

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java

@@ -25,6 +25,7 @@ import ca.uhn.fhir.batch2.model.WorkChunk;
 import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum;
 import ca.uhn.fhir.util.Logs;
 import ca.uhn.fhir.util.StopWatch;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.slf4j.Logger;
 
@@ -125,7 +126,7 @@ public class InstanceProgress {
 	public void updateInstance(JobInstance theInstance) {
 		updateInstance(theInstance, false);
 
-		String newWarningMessage = String.join("\n", myWarningMessages);
+		String newWarningMessage = StringUtils.right(String.join("\n", myWarningMessages), 4000);
 		theInstance.setWarningMessages(newWarningMessage);
 	}