Skip versions tested

This commit is contained in:
Ken Stevens 2019-12-14 19:00:58 -05:00
parent 4946fffac2
commit 7c3514fecc
15 changed files with 193 additions and 79 deletions

View File

@ -20,19 +20,16 @@ package ca.uhn.fhir.cli;
* #L%
*/
import ca.uhn.fhir.jpa.migrate.BaseMigrator;
import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
import ca.uhn.fhir.jpa.migrate.FlywayMigrator;
import ca.uhn.fhir.jpa.migrate.TaskOnlyMigrator;
import ca.uhn.fhir.jpa.migrate.*;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.defaultString;
@ -41,7 +38,7 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
* NB since 2019-12-05: This class is kind of weirdly named now, since it can either use Flyway or not use Flyway
*/
public abstract class BaseFlywayMigrateDatabaseCommand<T extends Enum> extends BaseCommand {
private static final Logger ourLog = LoggerFactory.getLogger(BaseFlywayMigrateDatabaseCommand.class);
public static final String MIGRATE_DATABASE = "migrate-database";
public static final String NO_COLUMN_SHRINK = "no-column-shrink";
@ -138,13 +135,18 @@ public abstract class BaseFlywayMigrateDatabaseCommand<T extends Enum> extends B
migrator.setDryRun(dryRun);
migrator.setNoColumnShrink(noColumnShrink);
migrator.setOutOfOrderPermitted(outOfOrderPermitted);
addTasks(migrator);
String skipVersions = theCommandLine.getOptionValue(BaseFlywayMigrateDatabaseCommand.SKIP_VERSIONS);
addTasks(migrator, skipVersions);
migrator.migrate();
}
protected abstract void addTasks(BaseMigrator theMigrator);
protected abstract void addTasks(BaseMigrator theMigrator, String theSkippedVersions);
public void setMigrationTableName(String theMigrationTableName) {
myMigrationTableName = theMigrationTableName;
}
protected void setDoNothingOnSkippedTasks(Collection<BaseTask> theTasks, String theSkipVersions) {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(theTasks, theSkipVersions);
}
}

View File

@ -21,7 +21,6 @@ package ca.uhn.fhir.cli;
*/
import ca.uhn.fhir.jpa.migrate.BaseMigrator;
import ca.uhn.fhir.jpa.migrate.FlywayMigrator;
import ca.uhn.fhir.jpa.migrate.SchemaMigrator;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import ca.uhn.fhir.jpa.migrate.tasks.HapiFhirJpaMigrationTasks;
@ -45,8 +44,9 @@ public class HapiFlywayMigrateDatabaseCommand extends BaseFlywayMigrateDatabaseC
}
@Override
protected void addTasks(BaseMigrator theMigrator) {
List<BaseTask<?>> tasks = new HapiFhirJpaMigrationTasks(getFlags()).getAllTasks(VersionEnum.values());
protected void addTasks(BaseMigrator theMigrator, String theSkipVersions) {
List<BaseTask> tasks = new HapiFhirJpaMigrationTasks(getFlags()).getAllTasks(VersionEnum.values());
super.setDoNothingOnSkippedTasks(tasks, theSkipVersions);
theMigrator.addTasks(tasks);
}

View File

@ -21,14 +21,15 @@ package ca.uhn.fhir.jpa.migrate;
*/
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import org.flywaydb.core.api.MigrationInfoService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
public abstract class BaseMigrator {
public abstract class BaseMigrator implements IMigrator {
private static final Logger ourLog = LoggerFactory.getLogger(BaseMigrator.class);
private boolean myDryRun;
private boolean myNoColumnShrink;
@ -39,8 +40,6 @@ public abstract class BaseMigrator {
private String myPassword;
private List<BaseTask.ExecutedStatement> myExecutedStatements = new ArrayList<>();
public abstract void migrate();
public boolean isDryRun() {
return myDryRun;
}
@ -57,10 +56,6 @@ public abstract class BaseMigrator {
myNoColumnShrink = theNoColumnShrink;
}
public abstract Optional<MigrationInfoService> getMigrationInfo();
public abstract void addTasks(List<BaseTask<?>> theMigrationTasks);
public DriverTypeEnum getDriverType() {
return myDriverType;
}

View File

@ -62,7 +62,7 @@ public class FlywayMigrator extends BaseMigrator {
myMigrationTableName = theMigrationTableName;
}
public void addTask(BaseTask<?> theTask) {
public void addTask(BaseTask theTask) {
myTasks.add(new FlywayMigration(theTask, this));
}
@ -97,7 +97,7 @@ public class FlywayMigrator extends BaseMigrator {
}
@Override
public void addTasks(List<BaseTask<?>> theTasks) {
public void addTasks(List<BaseTask> theTasks) {
theTasks.forEach(this::addTask);
}

View File

@ -0,0 +1,16 @@
package ca.uhn.fhir.jpa.migrate;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import org.flywaydb.core.api.MigrationInfoService;
import java.util.List;
import java.util.Optional;
public interface IMigrator {
void migrate();
Optional<MigrationInfoService> getMigrationInfo();
void addTasks(List<BaseTask> theMigrationTasks);
}

View File

@ -0,0 +1,34 @@
package ca.uhn.fhir.jpa.migrate;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.commons.lang3.StringUtils.isBlank;
public class MigrationTaskSkipper {
private static final Logger ourLog = LoggerFactory.getLogger(MigrationTaskSkipper.class);
public static void setDoNothingOnSkippedTasks(Collection<BaseTask> theTasks, String theSkipVersions) {
if (isBlank(theSkipVersions) || theTasks.isEmpty()) {
return;
}
Set<String> skippedVersionSet = Stream.of(theSkipVersions.split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.collect(Collectors.toSet());
for (BaseTask task : theTasks) {
if (skippedVersionSet.contains(task.getFlywayVersion())) {
ourLog.info("Will skip {}: {}", task.getFlywayVersion(), task.getDescription());
task.setDoNothing(true);
}
}
}
}

View File

@ -79,7 +79,7 @@ public class Migrator {
myConnectionProperties = myDriverType.newConnectionProperties(myConnectionUrl, myUsername, myPassword);
try {
for (BaseTask<?> next : myTasks) {
for (BaseTask next : myTasks) {
next.setDriverType(myDriverType);
next.setConnectionProperties(myConnectionProperties);
next.setDryRun(myDryRun);
@ -125,7 +125,7 @@ public class Migrator {
}
public void addTasks(List<BaseTask<?>> theTasks) {
public void addTasks(List<BaseTask> theTasks) {
theTasks.forEach(this::addTask);
}

View File

@ -34,28 +34,22 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.commons.lang3.StringUtils.isBlank;
public class SchemaMigrator {
public static final String HAPI_FHIR_MIGRATION_TABLENAME = "FLY_HFJ_MIGRATION";
private static final Logger ourLog = LoggerFactory.getLogger(SchemaMigrator.class);
public static final String HAPI_FHIR_MIGRATION_TABLENAME = "FLY_HFJ_MIGRATION";
private final BasicDataSource myDataSource;
private final boolean mySkipValidation;
private final String myMigrationTableName;
private final List<BaseTask<?>> myMigrationTasks;
private final List<BaseTask> myMigrationTasks;
private boolean myDontUseFlyway;
private boolean myOutOfOrderPermitted;
private String mySkipVersions;
private DriverTypeEnum myDriverType;
/**
* Constructor
*/
public SchemaMigrator(String theMigrationTableName, BasicDataSource theDataSource, Properties jpaProperties, List<BaseTask<?>> theMigrationTasks) {
public SchemaMigrator(String theMigrationTableName, BasicDataSource theDataSource, Properties jpaProperties, List<BaseTask> theMigrationTasks) {
myDataSource = theDataSource;
myMigrationTableName = theMigrationTableName;
myMigrationTasks = theMigrationTasks;
@ -138,24 +132,4 @@ public class SchemaMigrator {
public void setDriverType(DriverTypeEnum theDriverType) {
myDriverType = theDriverType;
}
public SchemaMigrator setSkipVersions(String theSkipVersions) {
mySkipVersions = theSkipVersions;
setDoNothingOnSkippedTasks();
return this;
}
private void setDoNothingOnSkippedTasks() {
if (isBlank(mySkipVersions) || myMigrationTasks.isEmpty()) {
return;
}
Set<String> skippedVersionSet = Stream.of(mySkipVersions.split(",")).map(String::trim).collect(Collectors.toSet());
for (BaseTask task : myMigrationTasks) {
if (skippedVersionSet.contains(task.getFlywayVersion())) {
ourLog.info("Will skip {}: {}", task.getFlywayVersion(), task.getDescription());
task.setDoNothing(true);
}
}
}
}

View File

@ -38,13 +38,13 @@ import java.util.Optional;
public class TaskOnlyMigrator extends BaseMigrator {
private static final Logger ourLog = LoggerFactory.getLogger(TaskOnlyMigrator.class);
private List<BaseTask<?>> myTasks = new ArrayList<>();
private List<BaseTask> myTasks = new ArrayList<>();
@Override
public void migrate() {
DriverTypeEnum.ConnectionProperties connectionProperties = getDriverType().newConnectionProperties(getConnectionUrl(), getUsername(), getPassword());
for (BaseTask<?> next : myTasks) {
for (BaseTask next : myTasks) {
next.setDriverType(getDriverType());
next.setDryRun(isDryRun());
next.setNoColumnShrink(isNoColumnShrink());
@ -71,7 +71,7 @@ public class TaskOnlyMigrator extends BaseMigrator {
}
@Override
public void addTasks(List<BaseTask<?>> theMigrationTasks) {
public void addTasks(List<BaseTask> theMigrationTasks) {
myTasks.addAll(theMigrationTasks);
}
}

View File

@ -33,16 +33,16 @@ import java.util.Collection;
import java.util.List;
public class BaseMigrationTasks<T extends Enum> {
private Multimap<T, BaseTask<?>> myTasks = MultimapBuilder.hashKeys().arrayListValues().build();
private Multimap<T, BaseTask> myTasks = MultimapBuilder.hashKeys().arrayListValues().build();
MigrationVersion lastVersion;
@SuppressWarnings("unchecked")
public List<BaseTask<?>> getTasks(@Nonnull T theFrom, @Nonnull T theTo) {
public List<BaseTask> getTasks(@Nonnull T theFrom, @Nonnull T theTo) {
Validate.notNull(theFrom);
Validate.notNull(theTo);
Validate.isTrue(theFrom.ordinal() < theTo.ordinal(), "From version must be lower than to version");
List<BaseTask<?>> retVal = new ArrayList<>();
List<BaseTask> retVal = new ArrayList<>();
for (Object nextVersion : EnumUtils.getEnumList(theFrom.getClass())) {
if (((T) nextVersion).ordinal() <= theFrom.ordinal()) {
continue;
@ -51,7 +51,7 @@ public class BaseMigrationTasks<T extends Enum> {
continue;
}
Collection<BaseTask<?>> nextValues = myTasks.get((T) nextVersion);
Collection<BaseTask> nextValues = myTasks.get((T) nextVersion);
if (nextValues != null) {
retVal.addAll(nextValues);
}
@ -68,10 +68,10 @@ public class BaseMigrationTasks<T extends Enum> {
return new Builder(theRelease.name(), sink);
}
public List<BaseTask<?>> getAllTasks(T[] theVersionEnumValues) {
List<BaseTask<?>> retval = new ArrayList<>();
public List<BaseTask> getAllTasks(T[] theVersionEnumValues) {
List<BaseTask> retval = new ArrayList<>();
for (T nextVersion : theVersionEnumValues) {
Collection<BaseTask<?>> nextValues = myTasks.get(nextVersion);
Collection<BaseTask> nextValues = myTasks.get(nextVersion);
if (nextValues != null) {
validate(nextValues);
retval.addAll(nextValues);
@ -81,7 +81,7 @@ public class BaseMigrationTasks<T extends Enum> {
return retval;
}
void validate(Collection<BaseTask<?>> theTasks) {
void validate(Collection<BaseTask> theTasks) {
for (BaseTask task: theTasks) {
task.validateVersion();
String version = task.getFlywayVersion();
@ -96,6 +96,6 @@ public class BaseMigrationTasks<T extends Enum> {
}
public interface IAcceptsTasks {
void addTask(BaseTask<?> theTask);
void addTask(BaseTask theTask);
}
}

View File

@ -39,7 +39,7 @@ public class Builder {
return new BuilderWithTableName(myRelease, mySink, theTableName);
}
public void addTask(BaseTask<?> theTask) {
public void addTask(BaseTask theTask) {
mySink.addTask(theTask);
}
@ -122,7 +122,7 @@ public class Builder {
}
@Override
public void addTask(BaseTask<?> theTask) {
public void addTask(BaseTask theTask) {
if (theTask instanceof AddColumnTask) {
myTask.addAddColumnTask((AddColumnTask) theTask);
} else {
@ -202,7 +202,7 @@ public class Builder {
}
@Override
public void addTask(BaseTask<?> theTask) {
public void addTask(BaseTask theTask) {
((BaseTableTask<?>) theTask).setTableName(myTableName);
mySink.addTask(theTask);
}

View File

@ -0,0 +1,91 @@
package ca.uhn.fhir.jpa.migrate;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import ca.uhn.fhir.jpa.migrate.taskdef.DropIndexTask;
import org.jetbrains.annotations.NotNull;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
public class MigrationTaskSkipperTest {
public static final String RELEASE = "4_1_0";
public static final String DATE_PREFIX = "20191214.";
private static final String VERSION_PREFIX = RELEASE + "." + DATE_PREFIX;
private List<BaseTask> myTasks;
@Before
public void before() {
myTasks = new ArrayList<>();
myTasks.add(new DropIndexTask(RELEASE, DATE_PREFIX + 1));
myTasks.add(new DropIndexTask(RELEASE, DATE_PREFIX + 2));
myTasks.add(new DropIndexTask(RELEASE, DATE_PREFIX + 3));
myTasks.add(new DropIndexTask(RELEASE, DATE_PREFIX + 4));
myTasks.add(new DropIndexTask(RELEASE, DATE_PREFIX + 5));
myTasks.add(new DropIndexTask(RELEASE, DATE_PREFIX + 6));
}
@Test
public void skipNull() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, null);
assertSkipped(myTasks);
}
@Test
public void skipAll() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, makeSkipString(1, 2, 3, 4, 5, 6));
assertSkipped(myTasks, 1, 2, 3, 4, 5, 6);
}
@Test
public void skipOne() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, makeSkipString(4));
assertSkipped(myTasks, 4);
}
@Test
public void skipWeirdSpacing() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, " " + VERSION_PREFIX + 2 + " , " + VERSION_PREFIX + 3 + " ");
assertSkipped(myTasks, 2, 3);
}
@Test
public void testDoubleComma() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, VERSION_PREFIX + 2 + ",," + VERSION_PREFIX + 3);
assertSkipped(myTasks, 2, 3);
}
@Test
public void startComma() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, "," + VERSION_PREFIX + 2 + "," + VERSION_PREFIX + 3);
assertSkipped(myTasks, 2, 3);
}
@Test
public void endComma() {
MigrationTaskSkipper.setDoNothingOnSkippedTasks(myTasks, VERSION_PREFIX + 2 + "," + VERSION_PREFIX + 3 + ",");
assertSkipped(myTasks, 2, 3);
}
private String makeSkipString(Integer... theVersions) {
return integersToVersions(theVersions).collect(Collectors.joining(","));
}
private void assertSkipped(List<BaseTask> theTasks, Integer... theVersions) {
Set<String> expectedVersions = integersToVersions(theVersions).collect(Collectors.toSet());
Set<String> taskVersions = theTasks.stream().filter(BaseTask::isDoNothing).map(BaseTask::getFlywayVersion).collect(Collectors.toSet());
assertThat(taskVersions, equalTo(expectedVersions));
}
@NotNull
private Stream<String> integersToVersions(Integer[] theVersions) {
return Stream.of(theVersions).map(s -> VERSION_PREFIX + s);
}
}

View File

@ -2,6 +2,7 @@ package ca.uhn.fhir.jpa.migrate;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.jpa.migrate.taskdef.AddTableRawSqlTask;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTest;
import com.google.common.collect.ImmutableList;
import org.flywaydb.core.api.FlywayException;
@ -84,10 +85,11 @@ public class SchemaMigratorTest extends BaseTest {
taskD.setTableName("SOMETABLE_D");
taskD.addSql(DriverTypeEnum.H2_EMBEDDED, "create table SOMETABLE_D (PID bigint not null, TEXTCOL varchar(255))");
SchemaMigrator schemaMigrator = new SchemaMigrator(SchemaMigrator.HAPI_FHIR_MIGRATION_TABLENAME, getDataSource(), new Properties(), ImmutableList.of(taskA, taskB, taskC, taskD));
ImmutableList<BaseTask> taskList = ImmutableList.of(taskA, taskB, taskC, taskD);
MigrationTaskSkipper.setDoNothingOnSkippedTasks(taskList, "4_1_0.20191214.2, 4_1_0.20191214.4");
SchemaMigrator schemaMigrator = new SchemaMigrator(SchemaMigrator.HAPI_FHIR_MIGRATION_TABLENAME, getDataSource(), new Properties(), taskList);
schemaMigrator.setDriverType(DriverTypeEnum.H2_EMBEDDED);
schemaMigrator.setSkipVersions("4_1_0.20191214.2, 4_1_0.20191214.4");
schemaMigrator.migrate();
DriverTypeEnum.ConnectionProperties connectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(getDataSource().getUrl(), getDataSource().getUsername(), getDataSource().getPassword());

View File

@ -33,7 +33,7 @@ public class HashTest {
@Test
public void testCheckAllHashes() {
List<BaseTask<?>> tasks1 = new HapiFhirJpaMigrationTasks(Collections.emptySet()).getAllTasks(VersionEnum.values());
List<BaseTask> tasks1 = new HapiFhirJpaMigrationTasks(Collections.emptySet()).getAllTasks(VersionEnum.values());
Map<String, Integer> hashesByVersion = new HashMap<>();
for (BaseTask task : tasks1) {
String version = task.getFlywayVersion();
@ -41,7 +41,7 @@ public class HashTest {
hashesByVersion.put(version, task.hashCode());
}
List<BaseTask<?>> tasks2 = new HapiFhirJpaMigrationTasks(Collections.emptySet()).getAllTasks(VersionEnum.values());
List<BaseTask> tasks2 = new HapiFhirJpaMigrationTasks(Collections.emptySet()).getAllTasks(VersionEnum.values());
for (BaseTask task : tasks2) {
String version = task.getFlywayVersion();
int origHash = hashesByVersion.get(version);

View File

@ -17,7 +17,7 @@ public class BaseMigrationTasksTest {
@Test
public void testValidateCorrectOrder() {
MyMigrationTasks migrationTasks = new MyMigrationTasks();
List<BaseTask<?>> tasks = new ArrayList<>();
List<BaseTask> tasks = new ArrayList<>();
tasks.add(new DropTableTask("1", "20191029.1"));
tasks.add(new DropTableTask("1", "20191029.2"));
migrationTasks.validate(tasks);
@ -26,7 +26,7 @@ public class BaseMigrationTasksTest {
@Test
public void testValidateVersionWrongOrder() {
MyMigrationTasks migrationTasks = new MyMigrationTasks();
List<BaseTask<?>> tasks = new ArrayList<>();
List<BaseTask> tasks = new ArrayList<>();
tasks.add(new DropTableTask("1", "20191029.2"));
tasks.add(new DropTableTask("1", "20191029.1"));
try {
@ -40,7 +40,7 @@ public class BaseMigrationTasksTest {
@Test
public void testValidateSameVersion() {
MyMigrationTasks migrationTasks = new MyMigrationTasks();
List<BaseTask<?>> tasks = new ArrayList<>();
List<BaseTask> tasks = new ArrayList<>();
tasks.add(new DropTableTask("1", "20191029.1"));
tasks.add(new DropTableTask("1", "20191029.1"));
try {
@ -54,7 +54,7 @@ public class BaseMigrationTasksTest {
@Test
public void testValidateWrongDateOrder() {
MyMigrationTasks migrationTasks = new MyMigrationTasks();
List<BaseTask<?>> tasks = new ArrayList<>();
List<BaseTask> tasks = new ArrayList<>();
tasks.add(new DropTableTask("1", "20191029.1"));
tasks.add(new DropTableTask("1", "20191028.1"));
try {