flyway initial implementation (with FIXMEs)

hapi-fhir-cli/hapi-fhir-cli-api/pom.xml

@@ -248,6 +248,10 @@
 			<artifactId>jansi</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.flywaydb</groupId>
+			<artifactId>flyway-core</artifactId>
+		</dependency>
 	</dependencies>
 
 	<build>

hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/FlywayMigrateDatabaseCommand.java

@@ -0,0 +1,112 @@
+package ca.uhn.fhir.cli;
+
+/*-
+ * #%L
+ * HAPI FHIR - Command Line Client - API
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
+import ca.uhn.fhir.jpa.migrate.FlywayMigrator;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public abstract class FlywayMigrateDatabaseCommand<T extends Enum> extends BaseCommand {
+
+	private static final String FLYWAY_MIGRATE_DATABASE = "flyway-migrate-database";
+	private Set<String> myFlags;
+
+	protected Set<String> getFlags() {
+		return myFlags;
+	}
+
+	@Override
+	public String getCommandDescription() {
+		return "This command migrates a HAPI FHIR JPA database from one version of HAPI FHIR to a newer version";
+	}
+
+	protected abstract List<T> provideAllowedVersions();
+
+	protected abstract Class<T> provideVersionEnumType();
+
+	@Override
+	public String getCommandName() {
+		return FLYWAY_MIGRATE_DATABASE;
+	}
+
+	@Override
+	public List<String> provideUsageNotes() {
+		String versions = "The following versions are supported: " +
+			provideAllowedVersions().stream().map(Enum::name).collect(Collectors.joining(", "));
+		return Collections.singletonList(versions);
+	}
+
+	@Override
+	public Options getOptions() {
+		Options retVal = new Options();
+
+		addOptionalOption(retVal, "r", "dry-run", false, "Log the SQL statements that would be executed but to not actually make any changes");
+
+		addRequiredOption(retVal, "u", "url", "URL", "The JDBC database URL");
+		addRequiredOption(retVal, "n", "username", "Username", "The JDBC database username");
+		addRequiredOption(retVal, "p", "password", "Password", "The JDBC database password");
+		addRequiredOption(retVal, "d", "driver", "Driver", "The database driver to use (Options are " + driverOptions() + ")");
+		addOptionalOption(retVal, null, "no-column-shrink", false, "If this flag is set, the system will not attempt to reduce the length of columns. This is useful in environments with a lot of existing data, where shrinking a column can take a very long time.");
+
+		return retVal;
+	}
+
+	private String driverOptions() {
+		return Arrays.stream(DriverTypeEnum.values()).map(Enum::name).collect(Collectors.joining(", "));
+	}
+
+	@Override
+	public void run(CommandLine theCommandLine) throws ParseException {
+
+		String url = theCommandLine.getOptionValue("u");
+		String username = theCommandLine.getOptionValue("n");
+		String password = theCommandLine.getOptionValue("p");
+		DriverTypeEnum driverType;
+		String driverTypeString = theCommandLine.getOptionValue("d");
+		try {
+			driverType = DriverTypeEnum.valueOf(driverTypeString);
+		} catch (Exception e) {
+			throw new ParseException("Invalid driver type \"" + driverTypeString + "\". Valid values are: " + driverOptions());
+		}
+
+		boolean dryRun = theCommandLine.hasOption("r");
+		boolean noColumnShrink = theCommandLine.hasOption("no-column-shrink");
+
+		FlywayMigrator migrator = new FlywayMigrator();
+		migrator.setConnectionUrl(url);
+		migrator.setDriverType(driverType);
+		migrator.setUsername(username);
+		migrator.setPassword(password);
+		migrator.setDryRun(dryRun);
+		migrator.setNoColumnShrink(noColumnShrink);
+
+		migrator.migrate();
+	}
+
+}

hapi-fhir-jpaserver-migrate/pom.xml

@@ -75,6 +75,11 @@
 			<artifactId>annotations</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.flywaydb</groupId>
+			<artifactId>flyway-core</artifactId>
+		</dependency>
+
 	</dependencies>
 
 	<build>

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/DriverTypeEnum.java

@@ -99,7 +99,7 @@ public enum DriverTypeEnum {
 		return new ConnectionProperties(dataSource, txTemplate, this);
 	}
 
-	public static class ConnectionProperties {
+	public static class ConnectionProperties implements AutoCloseable {
 
 		private final DriverTypeEnum myDriverType;
 		private final DataSource myDataSource;
@@ -139,6 +139,7 @@ public enum DriverTypeEnum {
 			return myTxTemplate;
 		}
 
+		@Override
 		public void close() {
 			if (myDataSource instanceof DisposableBean) {
 				try {

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigration.java

@@ -0,0 +1,74 @@
+package ca.uhn.fhir.jpa.migrate;
+
+import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import org.flywaydb.core.api.MigrationVersion;
+import org.flywaydb.core.api.migration.Context;
+import org.flywaydb.core.api.migration.JavaMigration;
+
+import java.sql.SQLException;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+
+public class FlywayMigration implements JavaMigration {
+	private final BaseTask myTask;
+	private final FlywayMigrator myFlywayMigrator;
+	private DriverTypeEnum.ConnectionProperties myConnectionProperties;
+
+	public FlywayMigration(BaseTask theTask, FlywayMigrator theFlywayMigrator) {
+		myTask = theTask;
+		myFlywayMigrator = theFlywayMigrator;
+	}
+
+	@Override
+	public MigrationVersion getVersion() {
+		return MigrationVersion.fromVersion(myTask.getVersion());
+	}
+
+	@Override
+	public String getDescription() {
+		String retval = myTask.getDescription();
+		if (retval == null) {
+			retval = myTask.getClass().getSimpleName() + " " + getVersion();
+		}
+		return retval;
+	}
+
+	@Override
+	public Integer getChecksum() {
+		// FIXME KHS
+		return 0;
+	}
+
+	@Override
+	public boolean isUndo() {
+		return false;
+	}
+
+	@Override
+	public boolean canExecuteInTransaction() {
+		return false;
+	}
+
+	@Override
+	public void migrate(Context theContext) throws Exception {
+		myTask.setDriverType(myFlywayMigrator.getDriverType());
+		myTask.setDryRun(myFlywayMigrator.isDryRun());
+		myTask.setNoColumnShrink(myFlywayMigrator.isNoColumnShrink());
+		myTask.setConnectionProperties(myConnectionProperties);
+		try {
+			myTask.execute();
+		} catch (SQLException e) {
+			String description = myTask.getDescription();
+			if (isBlank(description)) {
+				description = myTask.getClass().getSimpleName();
+			}
+			String prefix = "Failure executing task \"" + description + "\", aborting! Cause: ";
+			throw new InternalErrorException(prefix + e.toString(), e);
+		}
+	}
+
+	public void setConnectionProperties(DriverTypeEnum.ConnectionProperties theConnectionProperties) {
+		myConnectionProperties = theConnectionProperties;
+	}
+}

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigrator.java

@@ -0,0 +1,127 @@
+package ca.uhn.fhir.jpa.migrate;
+
+/*-
+ * #%L
+ * HAPI FHIR JPA Server - Migration
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import com.google.common.annotations.VisibleForTesting;
+import org.flywaydb.core.Flyway;
+import org.flywaydb.core.api.MigrationVersion;
+import org.flywaydb.core.api.migration.Context;
+import org.flywaydb.core.api.migration.JavaMigration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+
+public class FlywayMigrator {
+
+	private static final Logger ourLog = LoggerFactory.getLogger(FlywayMigrator.class);
+	// FIXME KHS
+	private static int ourVersion = 0;
+
+	private DriverTypeEnum myDriverType;
+	private String myConnectionUrl;
+	private String myUsername;
+	private String myPassword;
+	private List<FlywayMigration> myTasks = new ArrayList<>();
+	private boolean myDryRun;
+	private boolean myNoColumnShrink;
+
+	public void setDriverType(DriverTypeEnum theDriverType) {
+		myDriverType = theDriverType;
+	}
+
+	public void setConnectionUrl(String theConnectionUrl) {
+		myConnectionUrl = theConnectionUrl;
+	}
+
+	public void setUsername(String theUsername) {
+		myUsername = theUsername;
+	}
+
+	public void setPassword(String thePassword) {
+		myPassword = thePassword;
+	}
+
+	public void addTask(BaseTask<?> theTask) {
+		if (theTask.getVersion() == null) {
+			theTask.setVersion("2." + (++ourVersion));
+		}
+		myTasks.add(new FlywayMigration(theTask, this));
+	}
+
+	public void setDryRun(boolean theDryRun) {
+		myDryRun = theDryRun;
+	}
+
+	public void migrate() {
+		try (DriverTypeEnum.ConnectionProperties connectionProperties = myDriverType.newConnectionProperties(myConnectionUrl, myUsername, myPassword)) {
+			Flyway flyway = Flyway.configure()
+				.dataSource(myConnectionUrl, myUsername, myPassword)
+				.baselineOnMigrate(true)
+				.javaMigrations(myTasks.toArray(new JavaMigration[0]))
+				.load();
+			for (FlywayMigration task : myTasks) {
+				task.setConnectionProperties(connectionProperties);
+			}
+			flyway.migrate();
+		}
+	}
+
+	public void addTasks(List<BaseTask<?>> theTasks) {
+		theTasks.forEach(this::addTask);
+	}
+
+
+	public void setNoColumnShrink(boolean theNoColumnShrink) {
+		myNoColumnShrink = theNoColumnShrink;
+	}
+
+	public DriverTypeEnum getDriverType() {
+		return myDriverType;
+	}
+
+	public String getConnectionUrl() {
+		return myConnectionUrl;
+	}
+
+	public String getUsername() {
+		return myUsername;
+	}
+
+	public String getPassword() {
+		return myPassword;
+	}
+
+	public boolean isDryRun() {
+		return myDryRun;
+	}
+
+	public boolean isNoColumnShrink() {
+		return myNoColumnShrink;
+	}
+}

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java

@@ -434,6 +434,9 @@ public class JdbcUtils {
 						if ("SYSTEM TABLE".equalsIgnoreCase(tableType)) {
 							continue;
 						}
+						if ("FLYWAY_SCHEMA_HISTORY".equalsIgnoreCase(tableName)) {
+							continue;
+						}
 
 						columnNames.add(tableName);
 					}

hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java

@@ -43,6 +43,8 @@ public abstract class BaseTask<T extends BaseTask> {
 	private boolean myDryRun;
 	private List<ExecutedStatement> myExecutedStatements = new ArrayList<>();
 	private boolean myNoColumnShrink;
+	// FIXME KHS final
+	private String version;
 
 	public boolean isNoColumnShrink() {
 		return myNoColumnShrink;
@@ -130,6 +132,15 @@ public abstract class BaseTask<T extends BaseTask> {
 
 	public abstract void execute() throws SQLException;
 
+	public String getVersion() {
+		return version;
+	}
+
+	public BaseTask<T> setVersion(String theVersion) {
+		version = theVersion;
+		return this;
+	}
+
 	public static class ExecutedStatement {
 		private final String mySql;
 		private final List<Object> myArguments;

hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTest.java

@@ -1,6 +1,7 @@
 package ca.uhn.fhir.jpa.migrate.taskdef;
 
 import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
+import ca.uhn.fhir.jpa.migrate.FlywayMigrator;
 import ca.uhn.fhir.jpa.migrate.Migrator;
 import org.intellij.lang.annotations.Language;
 import org.junit.After;
@@ -14,7 +15,7 @@ public class BaseTest {
 
 	private static int ourDatabaseUrl = 0;
 	private String myUrl;
-	private Migrator myMigrator;
+	private FlywayMigrator myMigrator;
 	private DriverTypeEnum.ConnectionProperties myConnectionProperties;
 
 	public String getUrl() {
@@ -25,6 +26,10 @@ public class BaseTest {
 		return myConnectionProperties;
 	}
 
+	@After
+	public void resetMigrationVersion() {
+			executeSql("DELETE from \"flyway_schema_history\" where \"installed_rank\" > 0");
+	}
 
 	protected void executeSql(@Language("SQL") String theSql, Object... theArgs) {
 		myConnectionProperties.getTxTemplate().execute(t -> {
@@ -39,7 +44,7 @@ public class BaseTest {
 		});
 	}
 
-	public Migrator getMigrator() {
+	public FlywayMigrator getMigrator() {
 		return myMigrator;
 	}
 
@@ -56,7 +61,7 @@ public class BaseTest {
 
 		myConnectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(myUrl, "SA", "SA");
 
-		myMigrator = new Migrator();
+		myMigrator = new FlywayMigrator();
 		myMigrator.setConnectionUrl(myUrl);
 		myMigrator.setDriverType(DriverTypeEnum.H2_EMBEDDED);
 		myMigrator.setUsername("SA");

hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.migrate.taskdef;
 
 import ca.uhn.fhir.jpa.migrate.JdbcUtils;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import org.flywaydb.core.api.FlywayException;
 import org.junit.Test;
 
 import java.sql.SQLException;
@@ -63,8 +64,8 @@ public class RenameColumnTaskTest extends BaseTest {
 		try {
 			getMigrator().migrate();
 			fail();
-		} catch (InternalErrorException e) {
+		} catch (FlywayException e) {
-			assertEquals("Failure executing task \"Drop an index\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because both columns exist and data exists in TEXTCOL", e.getMessage());
+			assertEquals("Failure executing task \"Drop an index\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because both columns exist and data exists in TEXTCOL", e.getCause().getCause().getMessage());
 		}
 
 	}
@@ -98,8 +99,8 @@ public class RenameColumnTaskTest extends BaseTest {
 		try {
 			getMigrator().migrate();
 			fail();
-		} catch (InternalErrorException e) {
+		} catch (FlywayException e) {
-			assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because neither column exists!", e.getMessage());
+			assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because neither column exists!", e.getCause().getCause().getMessage());
 		}
 
 
@@ -132,8 +133,8 @@ public class RenameColumnTaskTest extends BaseTest {
 		try {
 			getMigrator().migrate();
 			fail();
-		} catch (InternalErrorException e) {
+		} catch (FlywayException e) {
-			assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.PID to PID2 because both columns exist!", e.getMessage());
+			assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.PID to PID2 because both columns exist!", e.getCause().getCause().getMessage());
 		}
 
 

pom.xml

@@ -1415,6 +1415,11 @@
 				<artifactId>xpp3_xpath</artifactId>
 				<version>1.1.4c</version>
 			</dependency>
+			<dependency>
+				<groupId>org.flywaydb</groupId>
+				<artifactId>flyway-core</artifactId>
+				<version>6.0.4</version>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>