removed old migration command

This commit is contained in:
Ken Stevens 2019-10-30 17:32:46 -04:00
parent 0501f13e66
commit d29e112e26
6 changed files with 4 additions and 569 deletions

View File

@ -161,7 +161,6 @@ public abstract class BaseApp {
commands.add(new IgPackUploader());
commands.add(new ExportConceptMapToCsvCommand());
commands.add(new ImportCsvToConceptMapCommand());
commands.add(new HapiMigrateDatabaseCommand());
commands.add(new HapiFlywayMigrateDatabaseCommand());
return commands;
}

View File

@ -37,7 +37,7 @@ import static org.apache.commons.lang3.StringUtils.defaultString;
public abstract class BaseFlywayMigrateDatabaseCommand<T extends Enum> extends BaseCommand {
private static final String FLYWAY_MIGRATE_DATABASE = "flyway-migrate-database";
public static final String MIGRATE_DATABASE = "migrate-database";
private Set<String> myFlags;
protected Set<String> getFlags() {
@ -46,7 +46,7 @@ public abstract class BaseFlywayMigrateDatabaseCommand<T extends Enum> extends B
@Override
public String getCommandDescription() {
return "This command migrates a HAPI FHIR JPA database from one version of HAPI FHIR to a newer version";
return "This command migrates a HAPI FHIR JPA database to the current version";
}
protected abstract List<T> provideAllowedVersions();
@ -55,7 +55,7 @@ public abstract class BaseFlywayMigrateDatabaseCommand<T extends Enum> extends B
@Override
public String getCommandName() {
return FLYWAY_MIGRATE_DATABASE;
return MIGRATE_DATABASE;
}
@Override

View File

@ -1,137 +0,0 @@
package ca.uhn.fhir.cli;
/*-
* #%L
* HAPI FHIR - Command Line Client - API
* %%
* Copyright (C) 2014 - 2019 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
import ca.uhn.fhir.jpa.migrate.Migrator;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.defaultString;
public abstract class BaseMigrateDatabaseCommand<T extends Enum> extends BaseCommand {
private static final String MIGRATE_DATABASE = "migrate-database";
private Set<String> myFlags;
protected Set<String> getFlags() {
return myFlags;
}
@Override
public String getCommandDescription() {
return "This command migrates a HAPI FHIR JPA database from one version of HAPI FHIR to a newer version";
}
protected abstract List<T> provideAllowedVersions();
protected abstract Class<T> provideVersionEnumType();
@Override
public String getCommandName() {
return MIGRATE_DATABASE;
}
@Override
public List<String> provideUsageNotes() {
String versions = "The following versions are supported: " +
provideAllowedVersions().stream().map(Enum::name).collect(Collectors.joining(", "));
return Collections.singletonList(versions);
}
@Override
public Options getOptions() {
Options retVal = new Options();
addOptionalOption(retVal, "r", "dry-run", false, "Log the SQL statements that would be executed but to not actually make any changes");
addRequiredOption(retVal, "u", "url", "URL", "The JDBC database URL");
addRequiredOption(retVal, "n", "username", "Username", "The JDBC database username");
addRequiredOption(retVal, "p", "password", "Password", "The JDBC database password");
addRequiredOption(retVal, "f", "from", "Version", "The database schema version to migrate FROM");
addRequiredOption(retVal, "t", "to", "Version", "The database schema version to migrate TO");
addRequiredOption(retVal, "d", "driver", "Driver", "The database driver to use (Options are " + driverOptions() + ")");
addOptionalOption(retVal, "x", "flags", "Flags", "A comma-separated list of any specific migration flags (these flags are version specific, see migrator documentation for details)");
addOptionalOption(retVal, null, "no-column-shrink", false, "If this flag is set, the system will not attempt to reduce the length of columns. This is useful in environments with a lot of existing data, where shrinking a column can take a very long time.");
return retVal;
}
private String driverOptions() {
return Arrays.stream(DriverTypeEnum.values()).map(Enum::name).collect(Collectors.joining(", "));
}
@Override
public void run(CommandLine theCommandLine) throws ParseException {
String url = theCommandLine.getOptionValue("u");
String username = theCommandLine.getOptionValue("n");
String password = theCommandLine.getOptionValue("p");
DriverTypeEnum driverType;
String driverTypeString = theCommandLine.getOptionValue("d");
try {
driverType = DriverTypeEnum.valueOf(driverTypeString);
} catch (Exception e) {
throw new ParseException("Invalid driver type \"" + driverTypeString + "\". Valid values are: " + driverOptions());
}
T from = getAndParseOptionEnum(theCommandLine, "f", provideVersionEnumType(), true, null);
validateVersionSupported(from);
T to = getAndParseOptionEnum(theCommandLine, "t", provideVersionEnumType(), true, null);
validateVersionSupported(to);
boolean dryRun = theCommandLine.hasOption("r");
boolean noColumnShrink = theCommandLine.hasOption("no-column-shrink");
String flags = theCommandLine.getOptionValue("x");
myFlags = Arrays.stream(defaultString(flags).split(","))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.collect(Collectors.toSet());
Migrator migrator = new Migrator();
migrator.setConnectionUrl(url);
migrator.setDriverType(driverType);
migrator.setUsername(username);
migrator.setPassword(password);
migrator.setDryRun(dryRun);
migrator.setNoColumnShrink(noColumnShrink);
addTasks(migrator, from, to);
migrator.migrate();
}
private void validateVersionSupported(T theFrom) throws ParseException {
if (provideAllowedVersions().contains(theFrom) == false) {
throw new ParseException("The version " + theFrom + " is not supported for migration");
}
}
protected abstract void addTasks(Migrator theMigrator, T theFrom, T theTo);
}

View File

@ -1,48 +0,0 @@
package ca.uhn.fhir.cli;
/*-
* #%L
* HAPI FHIR - Command Line Client - API
* %%
* Copyright (C) 2014 - 2019 University Health Network
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import ca.uhn.fhir.jpa.migrate.Migrator;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import ca.uhn.fhir.jpa.migrate.tasks.HapiFhirJpaMigrationTasks;
import ca.uhn.fhir.util.VersionEnum;
import java.util.Arrays;
import java.util.List;
public class HapiMigrateDatabaseCommand extends BaseMigrateDatabaseCommand<VersionEnum> {
@Override
protected List<VersionEnum> provideAllowedVersions() {
return Arrays.asList(VersionEnum.values());
}
@Override
protected Class<VersionEnum> provideVersionEnumType() {
return VersionEnum.class;
}
@Override
protected void addTasks(Migrator theMigrator, VersionEnum theFrom, VersionEnum theTo) {
List<BaseTask<?>> tasks = new HapiFhirJpaMigrationTasks(getFlags()).getTasks(theFrom, theTo);
tasks.forEach(theMigrator::addTask);
}
}

View File

@ -55,7 +55,7 @@ public class HapiFlywayMigrateDatabaseCommandTest {
ourLog.info("**********************************************");
String[] args = new String[]{
"flyway-migrate-database",
BaseFlywayMigrateDatabaseCommand.MIGRATE_DATABASE,
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",

View File

@ -1,379 +0,0 @@
package ca.uhn.fhir.cli;
import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
import ca.uhn.fhir.util.VersionEnum;
import com.google.common.base.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.support.AbstractLobCreatingPreparedStatementCallback;
import org.springframework.jdbc.support.lob.DefaultLobHandler;
import org.springframework.jdbc.support.lob.LobCreator;
import javax.validation.constraints.NotNull;
import java.io.File;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class HapiMigrateDatabaseCommandTest {
private static final Logger ourLog = LoggerFactory.getLogger(HapiMigrateDatabaseCommandTest.class);
public static final String DB_DIRECTORY = "target/h2_test";
static {
System.setProperty("test", "true");
}
@Test
public void testMigrate_340_current() throws IOException {
File location = getLocation("migrator_h2_test_340_current");
String url = "jdbc:h2:" + location.getAbsolutePath() + ";create=true";
DriverTypeEnum.ConnectionProperties connectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(url, "", "");
String initSql = "/persistence_create_h2_340.sql";
executeSqlStatements(connectionProperties, initSql);
seedDatabase340(connectionProperties);
ourLog.info("**********************************************");
ourLog.info("Done Setup, Starting Migration...");
ourLog.info("**********************************************");
String[] args = new String[]{
"migrate-database",
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",
"-p", "",
"-f", "V3_4_0",
"-t", VersionEnum.latestVersion().toString()
};
App.main(args);
connectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = connectionProperties.newJdbcTemplate();
List<Map<String, Object>> values = jdbcTemplate.queryForList("SELECT * FROM hfj_spidx_token");
assertEquals(1, values.size());
assertEquals("identifier", values.get(0).get("SP_NAME"));
assertEquals("12345678", values.get(0).get("SP_VALUE"));
assertTrue(values.get(0).keySet().contains("HASH_IDENTITY"));
assertEquals(7001889285610424179L, values.get(0).get("HASH_IDENTITY"));
return null;
});
}
@NotNull
private File getLocation(String theDatabaseName) throws IOException {
File directory = new File(DB_DIRECTORY);
if (directory.exists()) {
FileUtils.deleteDirectory(directory);
}
return new File(DB_DIRECTORY + "/" + theDatabaseName);
}
@Test
public void testMigrate_340_370() throws IOException {
File location = getLocation("migrator_h2_test_340_360");
String url = "jdbc:h2:" + location.getAbsolutePath() + ";create=true";
DriverTypeEnum.ConnectionProperties connectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(url, "", "");
String initSql = "/persistence_create_h2_340.sql";
executeSqlStatements(connectionProperties, initSql);
seedDatabase340(connectionProperties);
ourLog.info("**********************************************");
ourLog.info("Done Setup, Starting Migration...");
ourLog.info("**********************************************");
String[] args = new String[]{
"migrate-database",
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",
"-p", "",
"-f", "V3_4_0",
"-t", "V3_7_0"
};
App.main(args);
connectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = connectionProperties.newJdbcTemplate();
List<Map<String, Object>> values = jdbcTemplate.queryForList("SELECT * FROM hfj_spidx_token");
assertEquals(1, values.size());
assertEquals("identifier", values.get(0).get("SP_NAME"));
assertEquals("12345678", values.get(0).get("SP_VALUE"));
assertTrue(values.get(0).keySet().contains("HASH_IDENTITY"));
assertEquals(7001889285610424179L, values.get(0).get("HASH_IDENTITY"));
return null;
});
}
@Test
public void testMigrate_340_350() throws IOException {
File location = getLocation("migrator_h2_test_340_350");
String url = "jdbc:h2:" + location.getAbsolutePath() + ";create=true";
DriverTypeEnum.ConnectionProperties connectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(url, "", "");
String initSql = "/persistence_create_h2_340.sql";
executeSqlStatements(connectionProperties, initSql);
seedDatabase340(connectionProperties);
ourLog.info("**********************************************");
ourLog.info("Done Setup, Starting Dry Run...");
ourLog.info("**********************************************");
String[] args = new String[]{
"migrate-database",
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",
"-p", "",
"-r",
"-f", "V3_4_0",
"-t", "V3_5_0"
};
App.main(args);
connectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = connectionProperties.newJdbcTemplate();
List<Map<String, Object>> values = jdbcTemplate.queryForList("SELECT * FROM hfj_spidx_token");
assertFalse(values.get(0).keySet().contains("HASH_IDENTITY"));
return null;
});
ourLog.info("**********************************************");
ourLog.info("Done Setup, Starting Migration...");
ourLog.info("**********************************************");
args = new String[]{
"migrate-database",
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",
"-p", "",
"-f", "V3_4_0",
"-t", "V3_5_0"
};
App.main(args);
connectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = connectionProperties.newJdbcTemplate();
List<Map<String, Object>> values = jdbcTemplate.queryForList("SELECT * FROM hfj_spidx_token");
assertEquals(1, values.size());
assertEquals("identifier", values.get(0).get("SP_NAME"));
assertEquals("12345678", values.get(0).get("SP_VALUE"));
assertTrue(values.get(0).keySet().contains("HASH_IDENTITY"));
assertEquals(7001889285610424179L, values.get(0).get("HASH_IDENTITY"));
return null;
});
}
private void seedDatabase340(DriverTypeEnum.ConnectionProperties theConnectionProperties) {
theConnectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = theConnectionProperties.newJdbcTemplate();
jdbcTemplate.execute(
"insert into HFJ_RESOURCE (RES_DELETED_AT, RES_VERSION, FORCED_ID_PID, HAS_TAGS, RES_PUBLISHED, RES_UPDATED, SP_HAS_LINKS, HASH_SHA256, SP_INDEX_STATUS, RES_LANGUAGE, SP_CMPSTR_UNIQ_PRESENT, SP_COORDS_PRESENT, SP_DATE_PRESENT, SP_NUMBER_PRESENT, SP_QUANTITY_PRESENT, SP_STRING_PRESENT, SP_TOKEN_PRESENT, SP_URI_PRESENT, RES_PROFILE, RES_TYPE, RES_VER, RES_ID) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
new AbstractLobCreatingPreparedStatementCallback(new DefaultLobHandler()) {
@Override
protected void setValues(PreparedStatement thePs, LobCreator theLobCreator) throws SQLException {
thePs.setNull(1, Types.TIMESTAMP);
thePs.setString(2, "R4");
thePs.setNull(3, Types.BIGINT);
thePs.setBoolean(4, false);
thePs.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
thePs.setTimestamp(6, new Timestamp(System.currentTimeMillis()));
thePs.setBoolean(7, false);
thePs.setNull(8, Types.VARCHAR);
thePs.setLong(9, 1L);
thePs.setNull(10, Types.VARCHAR);
thePs.setBoolean(11, false);
thePs.setBoolean(12, false);
thePs.setBoolean(13, false);
thePs.setBoolean(14, false);
thePs.setBoolean(15, false);
thePs.setBoolean(16, false);
thePs.setBoolean(17, false);
thePs.setBoolean(18, false);
thePs.setNull(19, Types.VARCHAR);
thePs.setString(20, "Patient");
thePs.setLong(21, 1L);
thePs.setLong(22, 1L);
}
}
);
jdbcTemplate.execute(
"insert into HFJ_RES_VER (RES_DELETED_AT, RES_VERSION, FORCED_ID_PID, HAS_TAGS, RES_PUBLISHED, RES_UPDATED, RES_ENCODING, RES_TEXT, RES_ID, RES_TYPE, RES_VER, PID) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
new AbstractLobCreatingPreparedStatementCallback(new DefaultLobHandler()) {
@Override
protected void setValues(PreparedStatement thePs, LobCreator theLobCreator) throws SQLException {
thePs.setNull(1, Types.TIMESTAMP);
thePs.setString(2, "R4");
thePs.setNull(3, Types.BIGINT);
thePs.setBoolean(4, false);
thePs.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
thePs.setTimestamp(6, new Timestamp(System.currentTimeMillis()));
thePs.setString(7, "JSON");
theLobCreator.setBlobAsBytes(thePs, 8, "{\"resourceType\":\"Patient\"}".getBytes(Charsets.US_ASCII));
thePs.setLong(9, 1L);
thePs.setString(10, "Patient");
thePs.setLong(11, 1L);
thePs.setLong(12, 1L);
}
}
);
jdbcTemplate.execute(
"insert into HFJ_SPIDX_STRING (SP_MISSING, SP_NAME, RES_ID, RES_TYPE, SP_UPDATED, SP_VALUE_EXACT, SP_VALUE_NORMALIZED, SP_ID) values (?, ?, ?, ?, ?, ?, ?, ?)",
new AbstractLobCreatingPreparedStatementCallback(new DefaultLobHandler()) {
@Override
protected void setValues(PreparedStatement thePs, LobCreator theLobCreator) throws SQLException {
thePs.setBoolean(1, false);
thePs.setString(2, "given");
thePs.setLong(3, 1L); // res-id
thePs.setString(4, "Patient");
thePs.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
thePs.setString(6, "ROBERT");
thePs.setString(7, "Robert");
thePs.setLong(8, 1L);
}
}
);
jdbcTemplate.execute(
"insert into HFJ_SPIDX_TOKEN (SP_MISSING, SP_NAME, RES_ID, RES_TYPE, SP_UPDATED, SP_SYSTEM, SP_VALUE, SP_ID) values (?, ?, ?, ?, ?, ?, ?, ?)",
new AbstractLobCreatingPreparedStatementCallback(new DefaultLobHandler()) {
@Override
protected void setValues(PreparedStatement thePs, LobCreator theLobCreator) throws SQLException {
thePs.setBoolean(1, false);
thePs.setString(2, "identifier");
thePs.setLong(3, 1L); // res-id
thePs.setString(4, "Patient");
thePs.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
thePs.setString(6, "http://foo");
thePs.setString(7, "12345678");
thePs.setLong(8, 1L);
}
}
);
jdbcTemplate.execute(
"insert into HFJ_SPIDX_DATE (SP_MISSING, SP_NAME, RES_ID, RES_TYPE, SP_UPDATED, SP_VALUE_HIGH, SP_VALUE_LOW, SP_ID) values (?, ?, ?, ?, ?, ?, ?, ?)",
new AbstractLobCreatingPreparedStatementCallback(new DefaultLobHandler()) {
@Override
protected void setValues(PreparedStatement thePs, LobCreator theLobCreator) throws SQLException {
thePs.setBoolean(1, false);
thePs.setString(2, "birthdate");
thePs.setLong(3, 1L); // res-id
thePs.setString(4, "Patient");
thePs.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
thePs.setTimestamp(6, new Timestamp(1000000000L)); // value high
thePs.setTimestamp(7, new Timestamp(1000000000L)); // value low
thePs.setLong(8, 1L);
}
}
);
return null;
});
}
@Test
public void testMigrate_340_350_NoMigrateHashes() throws IOException {
File location = getLocation("migrator_h2_test_340_350_nmh");
String url = "jdbc:h2:" + location.getAbsolutePath() + ";create=true";
DriverTypeEnum.ConnectionProperties connectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(url, "", "");
String initSql = "/persistence_create_h2_340.sql";
executeSqlStatements(connectionProperties, initSql);
seedDatabase340(connectionProperties);
ourLog.info("**********************************************");
ourLog.info("Done Setup, Starting Migration...");
ourLog.info("**********************************************");
String[] args = new String[]{
"migrate-database",
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",
"-p", "",
"-f", "V3_4_0",
"-t", "V3_5_0",
"-x", "no-migrate-350-hashes"
};
App.main(args);
connectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = connectionProperties.newJdbcTemplate();
List<Map<String, Object>> values = jdbcTemplate.queryForList("SELECT * FROM hfj_spidx_token");
assertEquals(1, values.size());
assertEquals("identifier", values.get(0).get("SP_NAME"));
assertEquals("12345678", values.get(0).get("SP_VALUE"));
assertEquals(null, values.get(0).get("HASH_IDENTITY"));
return null;
});
}
private void executeSqlStatements(DriverTypeEnum.ConnectionProperties theConnectionProperties, String theInitSql) throws
IOException {
String script = IOUtils.toString(HapiMigrateDatabaseCommandTest.class.getResourceAsStream(theInitSql), Charsets.UTF_8);
List<String> scriptStatements = new ArrayList<>(Arrays.asList(script.split("\n")));
for (int i = 0; i < scriptStatements.size(); i++) {
String nextStatement = scriptStatements.get(i);
if (isBlank(nextStatement)) {
scriptStatements.remove(i);
i--;
continue;
}
nextStatement = nextStatement.trim();
while (nextStatement.endsWith(";")) {
nextStatement = nextStatement.substring(0, nextStatement.length() - 1);
}
scriptStatements.set(i, nextStatement);
}
theConnectionProperties.getTxTemplate().execute(t -> {
for (String next : scriptStatements) {
theConnectionProperties.newJdbcTemplate().execute(next);
}
return null;
});
}
}