From 5f2c88d9d93fc0dfa2ecb5f7e4575fde43483823 Mon Sep 17 00:00:00 2001 From: Nathan Doef Date: Wed, 22 Mar 2023 14:17:02 -0400 Subject: [PATCH] Scaffolding for testing with different db vendors (#4653) * scaffolding for testing with different db vendors * changelog * refactor * javadocs --- ...tests-with-different-database-vendors.yaml | 4 + hapi-fhir-jpaserver-test-utilities/pom.xml | 20 +++++ .../fhir/jpa/embedded/H2EmbeddedDatabase.java | 69 +++++++++++++++++ .../HapiEmbeddedDatabasesExtension.java | 71 ++++++++++++++++++ .../jpa/embedded/JpaEmbeddedDatabase.java | 57 ++++++++++++++ .../jpa/embedded/MsSqlEmbeddedDatabase.java | 75 +++++++++++++++++++ .../embedded/PostgresEmbeddedDatabase.java | 53 +++++++++++++ .../jpa/embedded/HapiSchemaMigrationTest.java | 54 +++++++++++++ 8 files changed, 403 insertions(+) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4654-scaffolding-for-tests-with-different-database-vendors.yaml create mode 100644 hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/H2EmbeddedDatabase.java create mode 100644 hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/HapiEmbeddedDatabasesExtension.java create mode 100644 hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/JpaEmbeddedDatabase.java create mode 100644 hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/MsSqlEmbeddedDatabase.java create mode 100644 hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/PostgresEmbeddedDatabase.java create mode 100644 hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/embedded/HapiSchemaMigrationTest.java diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4654-scaffolding-for-tests-with-different-database-vendors.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4654-scaffolding-for-tests-with-different-database-vendors.yaml new file mode 100644 index 00000000000..1074af9c860 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4654-scaffolding-for-tests-with-different-database-vendors.yaml @@ -0,0 +1,4 @@ +--- +type: add +issue: 4654 +title: "Add scaffolding for automated migration tests that use different database vendors." diff --git a/hapi-fhir-jpaserver-test-utilities/pom.xml b/hapi-fhir-jpaserver-test-utilities/pom.xml index 87f59488b1a..7acd9ef12a7 100644 --- a/hapi-fhir-jpaserver-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-test-utilities/pom.xml @@ -158,6 +158,26 @@ elasticsearch compile + + org.testcontainers + postgresql + 1.17.6 + compile + + + org.testcontainers + mssqlserver + 1.17.6 + compile + + + org.postgresql + postgresql + + + com.microsoft.sqlserver + mssql-jdbc + org.testcontainers testcontainers diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/H2EmbeddedDatabase.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/H2EmbeddedDatabase.java new file mode 100644 index 00000000000..a19851c1407 --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/H2EmbeddedDatabase.java @@ -0,0 +1,69 @@ +package ca.uhn.fhir.jpa.embedded; + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * For testing purposes. + *

+ * Embedded database that uses a {@link ca.uhn.fhir.jpa.migrate.DriverTypeEnum#H2_EMBEDDED} driver and a .h2 file in the target directory. + */ +public class H2EmbeddedDatabase extends JpaEmbeddedDatabase { + + private static final String SCHEMA_NAME = "test"; + private static final String USERNAME = "SA"; + private static final String PASSWORD = "SA"; + private static final String DATABASE_DIRECTORY = "target/h2-migration-tests/"; + + private String myUrl; + + public H2EmbeddedDatabase(){ + deleteDatabaseDirectoryIfExists(); + String databasePath = DATABASE_DIRECTORY + SCHEMA_NAME; + myUrl = "jdbc:h2:" + new File(databasePath).getAbsolutePath(); + super.initialize(DriverTypeEnum.H2_EMBEDDED, myUrl, USERNAME, PASSWORD); + } + + @Override + public void stop() { + deleteDatabaseDirectoryIfExists(); + } + + @Override + public void clearDatabase() { + dropTables(); + dropSequences(); + } + + private void deleteDatabaseDirectoryIfExists() { + File directory = new File(DATABASE_DIRECTORY); + if (directory.exists()) { + try { + FileUtils.deleteDirectory(directory); + } catch (IOException theE) { + throw new RuntimeException("Could not delete database directory: " + DATABASE_DIRECTORY); + } + } + } + + private void dropTables() { + List> tableResult = getJdbcTemplate().queryForList("SELECT TABLE_NAME FROM information_schema.tables WHERE TABLE_SCHEMA = 'PUBLIC'"); + for(Map result : tableResult){ + String tableName = result.get("TABLE_NAME").toString(); + getJdbcTemplate().execute(String.format("DROP TABLE %s CASCADE", tableName)); + } + } + + private void dropSequences() { + List> sequenceResult = getJdbcTemplate().queryForList("SELECT * FROM information_schema.sequences WHERE SEQUENCE_SCHEMA = 'PUBLIC'"); + for(Map sequence : sequenceResult){ + String sequenceName = sequence.get("SEQUENCE_NAME").toString(); + getJdbcTemplate().execute(String.format("DROP SEQUENCE %s", sequenceName)); + } + } +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/HapiEmbeddedDatabasesExtension.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/HapiEmbeddedDatabasesExtension.java new file mode 100644 index 00000000000..3b59bbd7852 --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/HapiEmbeddedDatabasesExtension.java @@ -0,0 +1,71 @@ +package ca.uhn.fhir.jpa.embedded; + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import com.google.common.collect.Sets; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; + +import javax.sql.DataSource; +import java.util.Set; +import java.util.stream.Stream; + +public class HapiEmbeddedDatabasesExtension implements AfterAllCallback { + + private final JpaEmbeddedDatabase myH2EmbeddedDatabase; + private final JpaEmbeddedDatabase myPostgresEmbeddedDatabase; + private final JpaEmbeddedDatabase myMsSqlEmbeddedDatabase; + // TODO add Oracle + + public HapiEmbeddedDatabasesExtension(){ + myH2EmbeddedDatabase = new H2EmbeddedDatabase(); + myPostgresEmbeddedDatabase = new PostgresEmbeddedDatabase(); + myMsSqlEmbeddedDatabase = new MsSqlEmbeddedDatabase(); + } + + @Override + public void afterAll(ExtensionContext theExtensionContext) throws Exception { + for(JpaEmbeddedDatabase database : getAllEmbeddedDatabases()){ + database.stop(); + } + } + + public JpaEmbeddedDatabase getEmbeddedDatabase(DriverTypeEnum theDriverType){ + switch (theDriverType) { + case H2_EMBEDDED: + return myH2EmbeddedDatabase; + case POSTGRES_9_4: + return myPostgresEmbeddedDatabase; + case MSSQL_2012: + return myMsSqlEmbeddedDatabase; + default: + throw new IllegalArgumentException("Driver type not supported: " + theDriverType); + } + } + + public void clearDatabases(){ + for(JpaEmbeddedDatabase database : getAllEmbeddedDatabases()){ + database.clearDatabase(); + } + } + + public DataSource getDataSource(DriverTypeEnum theDriverTypeEnum){ + return getEmbeddedDatabase(theDriverTypeEnum).getDataSource(); + } + + private Set getAllEmbeddedDatabases(){ + return Sets.newHashSet(myH2EmbeddedDatabase, myPostgresEmbeddedDatabase, myMsSqlEmbeddedDatabase); + } + + public static class DatabaseVendorProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext context) { + return Stream.of( + Arguments.of(DriverTypeEnum.H2_EMBEDDED), + Arguments.of(DriverTypeEnum.POSTGRES_9_4), + Arguments.of(DriverTypeEnum.MSSQL_2012) + ); + } + } +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/JpaEmbeddedDatabase.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/JpaEmbeddedDatabase.java new file mode 100644 index 00000000000..5f5235fa3a7 --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/JpaEmbeddedDatabase.java @@ -0,0 +1,57 @@ +package ca.uhn.fhir.jpa.embedded; + + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; + +/** + * For testing purposes. + *

+ * Provides embedded database functionality. Inheritors of this class will have access to a datasource and JDBC Template for executing queries. + * Inheritors must make a call to {@link JpaEmbeddedDatabase#initialize(DriverTypeEnum, String, String, String)} + * in their constructor and override abstract methods. + */ +public abstract class JpaEmbeddedDatabase { + + private DriverTypeEnum myDriverType; + private String myUsername; + private String myPassword; + private String myUrl; + private DriverTypeEnum.ConnectionProperties myConnectionProperties; + private JdbcTemplate myJdbcTemplate; + + public abstract void stop(); + public abstract void clearDatabase(); + + public void initialize(DriverTypeEnum theDriverType, String theUrl, String theUsername, String thePassword){ + myDriverType = theDriverType; + myUsername = theUsername; + myPassword = thePassword; + myUrl = theUrl; + myConnectionProperties = theDriverType.newConnectionProperties(theUrl, theUsername, thePassword); + myJdbcTemplate = myConnectionProperties.newJdbcTemplate(); + } + + public String getUsername() { + return myUsername; + } + + public String getPassword() { + return myPassword; + } + + public String getUrl() { + return myUrl; + } + + public JdbcTemplate getJdbcTemplate() { + return myJdbcTemplate; + } + + public DataSource getDataSource(){ + return myConnectionProperties.getDataSource(); + } + +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/MsSqlEmbeddedDatabase.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/MsSqlEmbeddedDatabase.java new file mode 100644 index 00000000000..c4ad8a63a69 --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/MsSqlEmbeddedDatabase.java @@ -0,0 +1,75 @@ +package ca.uhn.fhir.jpa.embedded; + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.utility.DockerImageName; + +import java.util.List; +import java.util.Map; + +/** + * For testing purposes. + *

+ * Embedded database that uses a {@link ca.uhn.fhir.jpa.migrate.DriverTypeEnum#MSSQL_2012} driver + * and a dockerized Testcontainer. + * @see MS SQL Server TestContainer + */ +public class MsSqlEmbeddedDatabase extends JpaEmbeddedDatabase { + + private final MSSQLServerContainer myContainer; + + public MsSqlEmbeddedDatabase(){ + DockerImageName msSqlImage = DockerImageName.parse("mcr.microsoft.com/azure-sql-edge:latest").asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server"); + myContainer = new MSSQLServerContainer(msSqlImage).acceptLicense(); + myContainer.start(); + super.initialize(DriverTypeEnum.MSSQL_2012, myContainer.getJdbcUrl(), myContainer.getUsername(), myContainer.getPassword()); + } + + @Override + public void stop() { + myContainer.stop(); + } + + @Override + public void clearDatabase() { + dropForeignKeys(); + dropRemainingConstraints(); + dropTables(); + dropSequences(); + } + + + private void dropForeignKeys() { + List> queryResults = getJdbcTemplate().queryForList("SELECT * FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE CONSTRAINT_TYPE = 'FOREIGN KEY'"); + for(Map row : queryResults) { + String tableName = row.get("TABLE_NAME").toString(); + String constraintName = row.get("CONSTRAINT_NAME").toString(); + getJdbcTemplate().execute(String.format("ALTER TABLE \"%s\" DROP CONSTRAINT \"%s\"", tableName, constraintName)); + } + } + + private void dropRemainingConstraints() { + List> queryResults = getJdbcTemplate().queryForList("SELECT * FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS"); + for(Map row : queryResults){ + String tableName = row.get("TABLE_NAME").toString(); + String constraintName = row.get("CONSTRAINT_NAME").toString(); + getJdbcTemplate().execute(String.format("ALTER TABLE \"%s\" DROP CONSTRAINT \"%s\"", tableName, constraintName)); + } + } + + private void dropTables() { + List> queryResults = getJdbcTemplate().queryForList("SELECT name FROM SYS.TABLES WHERE is_ms_shipped = 'false'"); + for(Map row : queryResults){ + String tableName = row.get("name").toString(); + getJdbcTemplate().execute(String.format("DROP TABLE \"%s\"", tableName)); + } + } + + private void dropSequences() { + List> queryResults = getJdbcTemplate().queryForList("SELECT name FROM SYS.SEQUENCES WHERE is_ms_shipped = 'false'"); + for(Map row : queryResults){ + String sequenceName = row.get("name").toString(); + getJdbcTemplate().execute(String.format("DROP SEQUENCE \"%s\"", sequenceName)); + } + } +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/PostgresEmbeddedDatabase.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/PostgresEmbeddedDatabase.java new file mode 100644 index 00000000000..59d1f13f747 --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/embedded/PostgresEmbeddedDatabase.java @@ -0,0 +1,53 @@ +package ca.uhn.fhir.jpa.embedded; + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +import java.util.List; +import java.util.Map; + +/** + * For testing purposes. + *

+ * Embedded database that uses a {@link ca.uhn.fhir.jpa.migrate.DriverTypeEnum#POSTGRES_9_4} driver + * and a dockerized Testcontainer. + * @see Postgres TestContainer + */ +public class PostgresEmbeddedDatabase extends JpaEmbeddedDatabase { + + private final PostgreSQLContainer myContainer; + + public PostgresEmbeddedDatabase(){ + myContainer = new PostgreSQLContainer(DockerImageName.parse("postgres:latest")); + myContainer.start(); + super.initialize(DriverTypeEnum.POSTGRES_9_4, myContainer.getJdbcUrl(), myContainer.getUsername(), myContainer.getPassword()); + } + + @Override + public void stop() { + myContainer.stop(); + } + + @Override + public void clearDatabase() { + dropTables(); + dropSequences(); + } + + private void dropTables() { + List> tableResult = getJdbcTemplate().queryForList("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"); + for(Map result : tableResult){ + String tableName = result.get("table_name").toString(); + getJdbcTemplate().execute(String.format("DROP TABLE \"%s\" CASCADE", tableName)); + } + } + + private void dropSequences() { + List> sequenceResult = getJdbcTemplate().queryForList("SELECT sequence_name FROM information_schema.sequences WHERE sequence_schema = 'public'"); + for(Map sequence : sequenceResult){ + String sequenceName = sequence.get("sequence_name").toString(); + getJdbcTemplate().execute(String.format("DROP SEQUENCE \"%s\" CASCADE", sequenceName)); + } + } +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/embedded/HapiSchemaMigrationTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/embedded/HapiSchemaMigrationTest.java new file mode 100644 index 00000000000..48020d081be --- /dev/null +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/embedded/HapiSchemaMigrationTest.java @@ -0,0 +1,54 @@ +package ca.uhn.fhir.jpa.embedded; + + +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; +import ca.uhn.fhir.jpa.migrate.HapiMigrationStorageSvc; +import ca.uhn.fhir.jpa.migrate.MigrationTaskList; +import ca.uhn.fhir.jpa.migrate.SchemaMigrator; +import ca.uhn.fhir.jpa.migrate.dao.HapiMigrationDao; +import ca.uhn.fhir.jpa.migrate.tasks.HapiFhirJpaMigrationTasks; +import ca.uhn.fhir.util.VersionEnum; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.util.Collections; +import java.util.Properties; + +import static ca.uhn.fhir.jpa.migrate.SchemaMigrator.HAPI_FHIR_MIGRATION_TABLENAME; + + +public class HapiSchemaMigrationTest { + + private static final Logger ourLog = LoggerFactory.getLogger(HapiSchemaMigrationTest.class); + public static final String TEST_SCHEMA_NAME = "test"; + + @RegisterExtension + static HapiEmbeddedDatabasesExtension myEmbeddedServersExtension = new HapiEmbeddedDatabasesExtension(); + + @AfterEach + public void afterEach(){ + myEmbeddedServersExtension.clearDatabases(); + } + + + @ParameterizedTest + @ArgumentsSource(HapiEmbeddedDatabasesExtension.DatabaseVendorProvider.class) + public void testMigration(DriverTypeEnum theDriverType){ + ourLog.info("Running hapi fhir migration tasks for {}", theDriverType); + + DataSource dataSource = myEmbeddedServersExtension.getDataSource(theDriverType); + HapiMigrationDao hapiMigrationDao = new HapiMigrationDao(dataSource, theDriverType, HAPI_FHIR_MIGRATION_TABLENAME); + HapiMigrationStorageSvc hapiMigrationStorageSvc = new HapiMigrationStorageSvc(hapiMigrationDao); + + MigrationTaskList migrationTasks = new HapiFhirJpaMigrationTasks(Collections.EMPTY_SET).getAllTasks(VersionEnum.values()); + SchemaMigrator schemaMigrator = new SchemaMigrator(TEST_SCHEMA_NAME, HAPI_FHIR_MIGRATION_TABLENAME, dataSource, new Properties(), migrationTasks, hapiMigrationStorageSvc); + schemaMigrator.setDriverType(theDriverType); + schemaMigrator.createMigrationTableIfRequired(); + schemaMigrator.migrate(); + } +}