From 790f655a9586fb1276120680a024336efd30b157 Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Sun, 6 Oct 2019 17:55:10 -0400 Subject: [PATCH] flyway initial implementation (with FIXMEs) --- hapi-fhir-cli/hapi-fhir-cli-api/pom.xml | 6 +- .../cli/FlywayMigrateDatabaseCommand.java | 112 +++++++++++++++ hapi-fhir-jpaserver-migrate/pom.xml | 5 + .../uhn/fhir/jpa/migrate/DriverTypeEnum.java | 3 +- .../uhn/fhir/jpa/migrate/FlywayMigration.java | 74 ++++++++++ .../uhn/fhir/jpa/migrate/FlywayMigrator.java | 127 ++++++++++++++++++ .../ca/uhn/fhir/jpa/migrate/JdbcUtils.java | 3 + .../fhir/jpa/migrate/taskdef/BaseTask.java | 11 ++ .../fhir/jpa/migrate/taskdef/BaseTest.java | 11 +- .../migrate/taskdef/RenameColumnTaskTest.java | 13 +- pom.xml | 5 + 11 files changed, 359 insertions(+), 11 deletions(-) create mode 100644 hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/FlywayMigrateDatabaseCommand.java create mode 100644 hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigration.java create mode 100644 hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigrator.java diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml index e52586c680f..5ac89219534 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml @@ -247,7 +247,11 @@ org.fusesource.jansi jansi - + + + org.flywaydb + flyway-core + diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/FlywayMigrateDatabaseCommand.java b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/FlywayMigrateDatabaseCommand.java new file mode 100644 index 00000000000..215512bec44 --- /dev/null +++ b/hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/FlywayMigrateDatabaseCommand.java @@ -0,0 +1,112 @@ +package ca.uhn.fhir.cli; + +/*- + * #%L + * HAPI FHIR - Command Line Client - API + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import ca.uhn.fhir.jpa.migrate.FlywayMigrator; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public abstract class FlywayMigrateDatabaseCommand extends BaseCommand { + + private static final String FLYWAY_MIGRATE_DATABASE = "flyway-migrate-database"; + private Set myFlags; + + protected Set getFlags() { + return myFlags; + } + + @Override + public String getCommandDescription() { + return "This command migrates a HAPI FHIR JPA database from one version of HAPI FHIR to a newer version"; + } + + protected abstract List provideAllowedVersions(); + + protected abstract Class provideVersionEnumType(); + + @Override + public String getCommandName() { + return FLYWAY_MIGRATE_DATABASE; + } + + @Override + public List provideUsageNotes() { + String versions = "The following versions are supported: " + + provideAllowedVersions().stream().map(Enum::name).collect(Collectors.joining(", ")); + return Collections.singletonList(versions); + } + + @Override + public Options getOptions() { + Options retVal = new Options(); + + addOptionalOption(retVal, "r", "dry-run", false, "Log the SQL statements that would be executed but to not actually make any changes"); + + addRequiredOption(retVal, "u", "url", "URL", "The JDBC database URL"); + addRequiredOption(retVal, "n", "username", "Username", "The JDBC database username"); + addRequiredOption(retVal, "p", "password", "Password", "The JDBC database password"); + addRequiredOption(retVal, "d", "driver", "Driver", "The database driver to use (Options are " + driverOptions() + ")"); + addOptionalOption(retVal, null, "no-column-shrink", false, "If this flag is set, the system will not attempt to reduce the length of columns. This is useful in environments with a lot of existing data, where shrinking a column can take a very long time."); + + return retVal; + } + + private String driverOptions() { + return Arrays.stream(DriverTypeEnum.values()).map(Enum::name).collect(Collectors.joining(", ")); + } + + @Override + public void run(CommandLine theCommandLine) throws ParseException { + + String url = theCommandLine.getOptionValue("u"); + String username = theCommandLine.getOptionValue("n"); + String password = theCommandLine.getOptionValue("p"); + DriverTypeEnum driverType; + String driverTypeString = theCommandLine.getOptionValue("d"); + try { + driverType = DriverTypeEnum.valueOf(driverTypeString); + } catch (Exception e) { + throw new ParseException("Invalid driver type \"" + driverTypeString + "\". Valid values are: " + driverOptions()); + } + + boolean dryRun = theCommandLine.hasOption("r"); + boolean noColumnShrink = theCommandLine.hasOption("no-column-shrink"); + + FlywayMigrator migrator = new FlywayMigrator(); + migrator.setConnectionUrl(url); + migrator.setDriverType(driverType); + migrator.setUsername(username); + migrator.setPassword(password); + migrator.setDryRun(dryRun); + migrator.setNoColumnShrink(noColumnShrink); + + migrator.migrate(); + } + +} diff --git a/hapi-fhir-jpaserver-migrate/pom.xml b/hapi-fhir-jpaserver-migrate/pom.xml index a8e1a3e4e31..76c1b392952 100644 --- a/hapi-fhir-jpaserver-migrate/pom.xml +++ b/hapi-fhir-jpaserver-migrate/pom.xml @@ -75,6 +75,11 @@ annotations + + org.flywaydb + flyway-core + + diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/DriverTypeEnum.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/DriverTypeEnum.java index 6b2699b2e0a..d676d0117ad 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/DriverTypeEnum.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/DriverTypeEnum.java @@ -99,7 +99,7 @@ public enum DriverTypeEnum { return new ConnectionProperties(dataSource, txTemplate, this); } - public static class ConnectionProperties { + public static class ConnectionProperties implements AutoCloseable { private final DriverTypeEnum myDriverType; private final DataSource myDataSource; @@ -139,6 +139,7 @@ public enum DriverTypeEnum { return myTxTemplate; } + @Override public void close() { if (myDataSource instanceof DisposableBean) { try { diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigration.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigration.java new file mode 100644 index 00000000000..2f5275cc0d8 --- /dev/null +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigration.java @@ -0,0 +1,74 @@ +package ca.uhn.fhir.jpa.migrate; + +import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import org.flywaydb.core.api.MigrationVersion; +import org.flywaydb.core.api.migration.Context; +import org.flywaydb.core.api.migration.JavaMigration; + +import java.sql.SQLException; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +public class FlywayMigration implements JavaMigration { + private final BaseTask myTask; + private final FlywayMigrator myFlywayMigrator; + private DriverTypeEnum.ConnectionProperties myConnectionProperties; + + public FlywayMigration(BaseTask theTask, FlywayMigrator theFlywayMigrator) { + myTask = theTask; + myFlywayMigrator = theFlywayMigrator; + } + + @Override + public MigrationVersion getVersion() { + return MigrationVersion.fromVersion(myTask.getVersion()); + } + + @Override + public String getDescription() { + String retval = myTask.getDescription(); + if (retval == null) { + retval = myTask.getClass().getSimpleName() + " " + getVersion(); + } + return retval; + } + + @Override + public Integer getChecksum() { + // FIXME KHS + return 0; + } + + @Override + public boolean isUndo() { + return false; + } + + @Override + public boolean canExecuteInTransaction() { + return false; + } + + @Override + public void migrate(Context theContext) throws Exception { + myTask.setDriverType(myFlywayMigrator.getDriverType()); + myTask.setDryRun(myFlywayMigrator.isDryRun()); + myTask.setNoColumnShrink(myFlywayMigrator.isNoColumnShrink()); + myTask.setConnectionProperties(myConnectionProperties); + try { + myTask.execute(); + } catch (SQLException e) { + String description = myTask.getDescription(); + if (isBlank(description)) { + description = myTask.getClass().getSimpleName(); + } + String prefix = "Failure executing task \"" + description + "\", aborting! Cause: "; + throw new InternalErrorException(prefix + e.toString(), e); + } + } + + public void setConnectionProperties(DriverTypeEnum.ConnectionProperties theConnectionProperties) { + myConnectionProperties = theConnectionProperties; + } +} diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigrator.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigrator.java new file mode 100644 index 00000000000..47b23e83548 --- /dev/null +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/FlywayMigrator.java @@ -0,0 +1,127 @@ +package ca.uhn.fhir.jpa.migrate; + +/*- + * #%L + * HAPI FHIR JPA Server - Migration + * %% + * Copyright (C) 2014 - 2019 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import com.google.common.annotations.VisibleForTesting; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.flywaydb.core.api.migration.Context; +import org.flywaydb.core.api.migration.JavaMigration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +public class FlywayMigrator { + + private static final Logger ourLog = LoggerFactory.getLogger(FlywayMigrator.class); + // FIXME KHS + private static int ourVersion = 0; + + private DriverTypeEnum myDriverType; + private String myConnectionUrl; + private String myUsername; + private String myPassword; + private List myTasks = new ArrayList<>(); + private boolean myDryRun; + private boolean myNoColumnShrink; + + public void setDriverType(DriverTypeEnum theDriverType) { + myDriverType = theDriverType; + } + + public void setConnectionUrl(String theConnectionUrl) { + myConnectionUrl = theConnectionUrl; + } + + public void setUsername(String theUsername) { + myUsername = theUsername; + } + + public void setPassword(String thePassword) { + myPassword = thePassword; + } + + public void addTask(BaseTask theTask) { + if (theTask.getVersion() == null) { + theTask.setVersion("2." + (++ourVersion)); + } + myTasks.add(new FlywayMigration(theTask, this)); + } + + public void setDryRun(boolean theDryRun) { + myDryRun = theDryRun; + } + + public void migrate() { + try (DriverTypeEnum.ConnectionProperties connectionProperties = myDriverType.newConnectionProperties(myConnectionUrl, myUsername, myPassword)) { + Flyway flyway = Flyway.configure() + .dataSource(myConnectionUrl, myUsername, myPassword) + .baselineOnMigrate(true) + .javaMigrations(myTasks.toArray(new JavaMigration[0])) + .load(); + for (FlywayMigration task : myTasks) { + task.setConnectionProperties(connectionProperties); + } + flyway.migrate(); + } + } + + public void addTasks(List> theTasks) { + theTasks.forEach(this::addTask); + } + + + public void setNoColumnShrink(boolean theNoColumnShrink) { + myNoColumnShrink = theNoColumnShrink; + } + + public DriverTypeEnum getDriverType() { + return myDriverType; + } + + public String getConnectionUrl() { + return myConnectionUrl; + } + + public String getUsername() { + return myUsername; + } + + public String getPassword() { + return myPassword; + } + + public boolean isDryRun() { + return myDryRun; + } + + public boolean isNoColumnShrink() { + return myNoColumnShrink; + } +} diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java index f1c889442be..11741180157 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java @@ -434,6 +434,9 @@ public class JdbcUtils { if ("SYSTEM TABLE".equalsIgnoreCase(tableType)) { continue; } + if ("FLYWAY_SCHEMA_HISTORY".equalsIgnoreCase(tableName)) { + continue; + } columnNames.add(tableName); } diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java index d8986ce325b..ca6a00e7092 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java @@ -43,6 +43,8 @@ public abstract class BaseTask { private boolean myDryRun; private List myExecutedStatements = new ArrayList<>(); private boolean myNoColumnShrink; + // FIXME KHS final + private String version; public boolean isNoColumnShrink() { return myNoColumnShrink; @@ -130,6 +132,15 @@ public abstract class BaseTask { public abstract void execute() throws SQLException; + public String getVersion() { + return version; + } + + public BaseTask setVersion(String theVersion) { + version = theVersion; + return this; + } + public static class ExecutedStatement { private final String mySql; private final List myArguments; diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTest.java index fc80b286fc1..772cf65c933 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTest.java @@ -1,6 +1,7 @@ package ca.uhn.fhir.jpa.migrate.taskdef; import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import ca.uhn.fhir.jpa.migrate.FlywayMigrator; import ca.uhn.fhir.jpa.migrate.Migrator; import org.intellij.lang.annotations.Language; import org.junit.After; @@ -14,7 +15,7 @@ public class BaseTest { private static int ourDatabaseUrl = 0; private String myUrl; - private Migrator myMigrator; + private FlywayMigrator myMigrator; private DriverTypeEnum.ConnectionProperties myConnectionProperties; public String getUrl() { @@ -25,6 +26,10 @@ public class BaseTest { return myConnectionProperties; } + @After + public void resetMigrationVersion() { + executeSql("DELETE from \"flyway_schema_history\" where \"installed_rank\" > 0"); + } protected void executeSql(@Language("SQL") String theSql, Object... theArgs) { myConnectionProperties.getTxTemplate().execute(t -> { @@ -39,7 +44,7 @@ public class BaseTest { }); } - public Migrator getMigrator() { + public FlywayMigrator getMigrator() { return myMigrator; } @@ -56,7 +61,7 @@ public class BaseTest { myConnectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(myUrl, "SA", "SA"); - myMigrator = new Migrator(); + myMigrator = new FlywayMigrator(); myMigrator.setConnectionUrl(myUrl); myMigrator.setDriverType(DriverTypeEnum.H2_EMBEDDED); myMigrator.setUsername("SA"); diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java index b9351563513..53898614052 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java @@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.migrate.taskdef; import ca.uhn.fhir.jpa.migrate.JdbcUtils; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import org.flywaydb.core.api.FlywayException; import org.junit.Test; import java.sql.SQLException; @@ -63,8 +64,8 @@ public class RenameColumnTaskTest extends BaseTest { try { getMigrator().migrate(); fail(); - } catch (InternalErrorException e) { - assertEquals("Failure executing task \"Drop an index\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because both columns exist and data exists in TEXTCOL", e.getMessage()); + } catch (FlywayException e) { + assertEquals("Failure executing task \"Drop an index\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because both columns exist and data exists in TEXTCOL", e.getCause().getCause().getMessage()); } } @@ -98,8 +99,8 @@ public class RenameColumnTaskTest extends BaseTest { try { getMigrator().migrate(); fail(); - } catch (InternalErrorException e) { - assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because neither column exists!", e.getMessage()); + } catch (FlywayException e) { + assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.myTextCol to TEXTCOL because neither column exists!", e.getCause().getCause().getMessage()); } @@ -132,8 +133,8 @@ public class RenameColumnTaskTest extends BaseTest { try { getMigrator().migrate(); fail(); - } catch (InternalErrorException e) { - assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.PID to PID2 because both columns exist!", e.getMessage()); + } catch (FlywayException e) { + assertEquals("Failure executing task \"RenameColumnTask\", aborting! Cause: java.sql.SQLException: Can not rename SOMETABLE.PID to PID2 because both columns exist!", e.getCause().getCause().getMessage()); } diff --git a/pom.xml b/pom.xml index c4887d24589..ac50accbc48 100755 --- a/pom.xml +++ b/pom.xml @@ -1415,6 +1415,11 @@ xpp3_xpath 1.1.4c + + org.flywaydb + flyway-core + 6.0.4 +