Merge remote-tracking branch 'origin/master' into ng_20201218_survivorship_poc

This commit is contained in:
Nick 2021-01-19 09:35:31 -05:00
commit f5e2086cfd
19 changed files with 233 additions and 13 deletions

View File

@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.junit.jupiter.api.Assertions.*;
@ -61,9 +62,44 @@ public class HapiFlywayMigrateDatabaseCommandTest {
"-n", "",
"-p", ""
};
assertFalse(JdbcUtils.getTableNames(connectionProperties).contains("HFJ_RES_REINDEX_JOB"));
// Verify that HFJ_SEARCH_PARM exists along with index and foreign key dependencies.
assertTrue(JdbcUtils.getTableNames(connectionProperties).contains("HFJ_SEARCH_PARM"));
Set<String> indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_SEARCH_PARM");
assertTrue(indexNames.contains("IDX_SEARCHPARM_RESTYPE_SPNAME"));
Set<String> foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_SEARCH_PARM", "HFJ_RES_PARAM_PRESENT");
assertTrue(foreignKeys.contains("FK_RESPARMPRES_SPID"));
// Verify that IDX_FORCEDID_TYPE_FORCEDID index exists on HFJ_FORCED_ID table
indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_FORCED_ID");
assertTrue(indexNames.contains("IDX_FORCEDID_TYPE_FORCEDID"));
// Verify that HFJ_RES_PARAM_PRESENT has column SP_ID
Set<String> columnNames = JdbcUtils.getColumnNames(connectionProperties, "HFJ_RES_PARAM_PRESENT");
assertTrue(columnNames.contains("SP_ID"));
// Verify that SEQ_SEARCHPARM_ID sequence exists
Set<String> seqNames = JdbcUtils.getSequenceNames(connectionProperties);
assertTrue(seqNames.contains("SEQ_SEARCHPARM_ID"));
// Verify that foreign key FK_SEARCHRES_RES on HFJ_SEARCH_RESULT exists
foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_RESOURCE", "HFJ_SEARCH_RESULT");
assertTrue(foreignKeys.contains("FK_SEARCHRES_RES"));
App.main(args);
assertTrue(JdbcUtils.getTableNames(connectionProperties).contains("HFJ_RES_REINDEX_JOB"));
// Verify that HFJ_SEARCH_PARM has been removed
assertFalse(JdbcUtils.getTableNames(connectionProperties).contains("HFJ_SEARCH_PARM"));
// Verify that IDX_FORCEDID_TYPE_FORCEDID index no longer exists on HFJ_FORCED_ID table
indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_FORCED_ID");
assertFalse(indexNames.contains("IDX_FORCEDID_TYPE_FORCEDID"));
// Verify that HFJ_RES_PARAM_PRESENT no longer has column SP_ID
columnNames = JdbcUtils.getColumnNames(connectionProperties, "HFJ_RES_PARAM_PRESENT");
assertFalse(columnNames.contains("SP_ID"));
// Verify that SEQ_SEARCHPARM_ID sequence no longer exists
seqNames = JdbcUtils.getSequenceNames(connectionProperties);
assertFalse(seqNames.contains("SEQ_SEARCHPARM_ID"));
// Verify that foreign key FK_SEARCHRES_RES on HFJ_SEARCH_RESULT no longer exists
foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_RESOURCE", "HFJ_SEARCH_RESULT");
assertFalse(foreignKeys.contains("FK_SEARCHRES_RES"));
connectionProperties.getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = connectionProperties.newJdbcTemplate();
@ -118,6 +154,75 @@ public class HapiFlywayMigrateDatabaseCommandTest {
});
}
@Test
public void testMigrateFrom340_dryRun() throws IOException, SQLException {
File location = getLocation("migrator_h2_test_340_dryrun");
String url = "jdbc:h2:" + location.getAbsolutePath() + ";create=true";
DriverTypeEnum.ConnectionProperties connectionProperties = DriverTypeEnum.H2_EMBEDDED.newConnectionProperties(url, "", "");
String initSql = "/persistence_create_h2_340.sql";
executeSqlStatements(connectionProperties, initSql);
seedDatabase340(connectionProperties);
ourLog.info("**********************************************");
ourLog.info("Done Setup, Starting Migration...");
ourLog.info("**********************************************");
String[] args = new String[]{
BaseFlywayMigrateDatabaseCommand.MIGRATE_DATABASE,
"-d", "H2_EMBEDDED",
"-u", url,
"-n", "",
"-p", "",
"-r"
};
// Verify that HFJ_SEARCH_PARM exists along with index and foreign key dependencies.
assertTrue(JdbcUtils.getTableNames(connectionProperties).contains("HFJ_SEARCH_PARM"));
Set<String> indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_SEARCH_PARM");
assertTrue(indexNames.contains("IDX_SEARCHPARM_RESTYPE_SPNAME"));
Set<String> foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_SEARCH_PARM", "HFJ_RES_PARAM_PRESENT");
assertTrue(foreignKeys.contains("FK_RESPARMPRES_SPID"));
// Verify that IDX_FORCEDID_TYPE_FORCEDID index exists on HFJ_FORCED_ID table
indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_FORCED_ID");
assertTrue(indexNames.contains("IDX_FORCEDID_TYPE_FORCEDID"));
// Verify that HFJ_RES_PARAM_PRESENT has column SP_ID
Set<String> columnNames = JdbcUtils.getColumnNames(connectionProperties, "HFJ_RES_PARAM_PRESENT");
assertTrue(columnNames.contains("SP_ID"));
// Verify that SEQ_SEARCHPARM_ID sequence exists
Set<String> seqNames = JdbcUtils.getSequenceNames(connectionProperties);
assertTrue(seqNames.contains("SEQ_SEARCHPARM_ID"));
// Verify that foreign key FK_SEARCHRES_RES on HFJ_SEARCH_RESULT exists
foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_RESOURCE", "HFJ_SEARCH_RESULT");
assertTrue(foreignKeys.contains("FK_SEARCHRES_RES"));
App.main(args);
// Verify that HFJ_SEARCH_PARM still exists along with index and foreign key dependencies.
assertTrue(JdbcUtils.getTableNames(connectionProperties).contains("HFJ_SEARCH_PARM"));
indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_SEARCH_PARM");
assertTrue(indexNames.contains("IDX_SEARCHPARM_RESTYPE_SPNAME"));
foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_SEARCH_PARM", "HFJ_RES_PARAM_PRESENT");
assertTrue(foreignKeys.contains("FK_RESPARMPRES_SPID"));
// Verify that IDX_FORCEDID_TYPE_FORCEDID index still exists on HFJ_FORCED_ID table
indexNames = JdbcUtils.getIndexNames(connectionProperties, "HFJ_FORCED_ID");
assertTrue(indexNames.contains("IDX_FORCEDID_TYPE_FORCEDID"));
// Verify that HFJ_RES_PARAM_PRESENT still has column SP_ID
columnNames = JdbcUtils.getColumnNames(connectionProperties, "HFJ_RES_PARAM_PRESENT");
assertTrue(columnNames.contains("SP_ID"));
// Verify that SEQ_SEARCHPARM_ID sequence still exists
seqNames = JdbcUtils.getSequenceNames(connectionProperties);
assertTrue(seqNames.contains("SEQ_SEARCHPARM_ID"));
// Verify that foreign key FK_SEARCHRES_RES on HFJ_SEARCH_RESULT still exists
foreignKeys = JdbcUtils.getForeignKeys(connectionProperties, "HFJ_RESOURCE", "HFJ_SEARCH_RESULT");
assertTrue(foreignKeys.contains("FK_SEARCHRES_RES"));
}
@Test
public void testMigrateFromEmptySchema() throws IOException, SQLException {

View File

@ -0,0 +1,5 @@
---
type: fix
issue: 2298
title: "A recent change inadvertently caused an issue with DB migration utility such that it would attempt to drop
indexes even when the `dry-run` option was specified. This has been fixed."

View File

@ -0,0 +1,4 @@
---
type: fix
issue: 2299
title: "The BulkExportDaoSvc bean was forgotten from the BaseConfig context. It has been added."

View File

@ -21,6 +21,7 @@ package ca.uhn.fhir.jpa.bulk.job;
*/
import ca.uhn.fhir.jpa.batch.processors.PidToIBaseResourceProcessor;
import ca.uhn.fhir.jpa.bulk.svc.BulkExportDaoSvc;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.batch.core.Job;
@ -54,6 +55,11 @@ public class BulkExportJobConfig {
@Autowired
private PidToIBaseResourceProcessor myPidToIBaseResourceProcessor;
@Bean
public BulkExportDaoSvc bulkExportDaoSvc() {
return new BulkExportDaoSvc();
}
@Bean
@Lazy
public Job bulkExportJob() {

View File

@ -50,10 +50,6 @@ public class BulkExportDaoSvc {
IBulkExportCollectionDao myBulkExportCollectionDao;
@Autowired
IBulkExportCollectionFileDao myBulkExportCollectionFileDao;
@Autowired
private FhirContext myFhirContext;
@Autowired
private DaoRegistry myDaoRegistry;
@Transactional
public void addFileToCollectionWithId(Long theCollectionEntityId, BulkExportCollectionFileEntity theFile) {
@ -65,7 +61,6 @@ public class BulkExportDaoSvc {
myBulkExportCollectionFileDao.saveAndFlush(theFile);
myBulkExportCollectionDao.saveAndFlush(exportCollectionEntity);
}
}
@Transactional

View File

@ -220,6 +220,7 @@ public abstract class BaseConfig {
return new BatchJobSubmitterImpl();
}
@Lazy
@Bean
public CascadingDeleteInterceptor cascadingDeleteInterceptor(FhirContext theFhirContext, DaoRegistry theDaoRegistry, IInterceptorBroadcaster theInterceptorBroadcaster) {

View File

@ -73,10 +73,6 @@ public class TestR4Config extends BaseJavaConfigR4 {
return new CircularQueueCaptureQueriesListener();
}
@Bean
public BulkExportDaoSvc bulkExportDaoSvc() {
return new BulkExportDaoSvc();
}
@Bean
public DataSource dataSource() {

View File

@ -19,6 +19,7 @@ import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import ca.uhn.fhir.util.TestUtil;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.DateTimeType;
@ -50,6 +51,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
@ExtendWith(SpringExtension.class)
@RequiresDocker
@ContextConfiguration(classes = {TestR4ConfigWithElasticsearchClient.class})
public class BaseR4SearchLastN extends BaseJpaTest {

View File

@ -11,6 +11,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import org.hamcrest.Matchers;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.Bundle;
@ -59,6 +60,7 @@ import ca.uhn.fhir.validation.FhirValidator;
import ca.uhn.fhir.validation.ValidationResult;
@ExtendWith(SpringExtension.class)
@RequiresDocker
@ContextConfiguration(classes = {TestR4ConfigWithElasticSearch.class})
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {

View File

@ -18,6 +18,7 @@ import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry;
import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import ca.uhn.fhir.util.TestUtil;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CodeSystem;
@ -45,6 +46,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
@ExtendWith(SpringExtension.class)
@RequiresDocker
@ContextConfiguration(classes = {TestR4ConfigWithElasticSearch.class})
public class FhirResourceDaoR4TerminologyElasticsearchIT extends BaseJpaTest {

View File

@ -14,6 +14,7 @@ import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.parser.IParser;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import com.google.common.base.Charsets;
import org.apache.commons.io.IOUtils;
import org.aspectj.lang.annotation.Before;
@ -47,6 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(SpringExtension.class)
@RequiresDocker
@ContextConfiguration(classes = {TestR4ConfigWithElasticsearchClient.class})
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class PersistObservationIndexedSearchParamLastNR4IT {

View File

@ -18,6 +18,7 @@ import ca.uhn.fhir.rest.param.TokenAndListParam;
import ca.uhn.fhir.rest.param.TokenOrListParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -27,6 +28,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
@ -57,7 +59,8 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(SpringExtension.class)
@ExtendWith({SpringExtension.class})
@RequiresDocker
@Testcontainers
public class LastNElasticsearchSvcMultipleObservationsIT {
@ -70,9 +73,12 @@ public class LastNElasticsearchSvcMultipleObservationsIT {
private final FhirContext myFhirContext = FhirContext.forCached(FhirVersionEnum.R4);
@Container
public static ElasticsearchContainer elasticsearchContainer = TestElasticsearchContainerHelper.getEmbeddedElasticSearch();
private ElasticsearchSvcImpl elasticsearchSvc;
@BeforeEach

View File

@ -1,5 +1,10 @@
package ca.uhn.fhir.jpa.search.lastn.config;
import com.github.dockerjava.api.exception.InternalServerErrorException;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import java.time.Duration;
@ -13,6 +18,7 @@ public class TestElasticsearchContainerHelper {
public static final String ELASTICSEARCH_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:" + ELASTICSEARCH_VERSION;
public static ElasticsearchContainer getEmbeddedElasticSearch() {
return new ElasticsearchContainer(ELASTICSEARCH_IMAGE)
.withStartupTimeout(Duration.of(300, SECONDS));
}

View File

@ -67,7 +67,8 @@ public class DropTableTask extends BaseTableTask {
theIndexTask
.setTableName(getTableName())
.setConnectionProperties(getConnectionProperties())
.setDriverType(getDriverType());
.setDriverType(getDriverType())
.setDryRun(isDryRun());
for (String nextIndex : indexNames) {
theIndexTask
.setIndexName(nextIndex)

View File

@ -22,10 +22,14 @@ package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.migrate.JdbcUtils;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import org.intellij.lang.annotations.Language;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.ColumnMapRowMapper;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class ModifyColumnTask extends BaseTableColumnTypeTask {
@ -56,7 +60,7 @@ public class ModifyColumnTask extends BaseTableColumnTypeTask {
try {
existingType = JdbcUtils.getColumnType(getConnectionProperties(), getTableName(), getColumnName());
nullable = JdbcUtils.isColumnNullable(getConnectionProperties(), getTableName(), getColumnName());
nullable = isColumnNullable(getTableName(), getColumnName());
} catch (SQLException e) {
throw new InternalErrorException(e);
}
@ -152,4 +156,38 @@ public class ModifyColumnTask extends BaseTableColumnTypeTask {
}
}
private boolean isColumnNullable(String tableName, String columnName) throws SQLException {
boolean result = JdbcUtils.isColumnNullable(getConnectionProperties(), tableName, columnName);
// Oracle sometimes stores the NULLABLE property in a Constraint, so override the result if this is an Oracle DB
switch (getDriverType()) {
case ORACLE_12C:
@Language("SQL") String findNullableConstraintSql =
"SELECT acc.owner, acc.table_name, acc.column_name, search_condition_vc " +
"FROM all_cons_columns acc, all_constraints ac " +
"WHERE acc.constraint_name = ac.constraint_name " +
"AND acc.table_name = ac.table_name " +
"AND ac.constraint_type = ? " +
"AND acc.table_name = ? " +
"AND acc.column_name = ? " +
"AND search_condition_vc = ? ";
String[] params = new String[4];
params[0] = "C";
params[1] = tableName.toUpperCase();
params[2] = columnName.toUpperCase();
params[3] = "\"" + columnName.toUpperCase() + "\" IS NOT NULL";
List<Map<String, Object>> queryResults = getConnectionProperties().getTxTemplate().execute(t -> {
return getConnectionProperties().newJdbcTemplate().query(findNullableConstraintSql, params, new ColumnMapRowMapper());
});
// If this query returns a row then the existence of that row indicates that a NOT NULL constraint exists
// on this Column and we must override whatever result was previously calculated and set it to false
if (queryResults != null && queryResults.size() > 0 && queryResults.get(0) != null && !queryResults.get(0).isEmpty()) {
result = false;
}
break;
default:
// Do nothing since we already initialized the variable above
break;
}
return result;
}
}

View File

@ -68,5 +68,5 @@
<artifactId>junit-jupiter-engine</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</dependencies>
</project>

View File

@ -95,6 +95,17 @@
<artifactId>junit-jupiter-engine</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

View File

@ -0,0 +1,23 @@
package ca.uhn.fhir.test.utilities.docker;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.DockerClientFactory;
/**
* Execution condition which will skip test classes that require docker if it is not present on the host machine
*/
public class DockerRequiredCondition implements ExecutionCondition {
@Override
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext theExtensionContext) {
try {
DockerClientFactory.instance().isDockerAvailable();
return ConditionEvaluationResult.enabled("Docker is installed so we can run these tests!");
} catch (Exception e) {
return ConditionEvaluationResult.disabled("It appears as though docker is not installed on the host machine!");
}
}
}

View File

@ -0,0 +1,15 @@
package ca.uhn.fhir.test.utilities.docker;
import org.junit.jupiter.api.extension.ExtendWith;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@ExtendWith(DockerRequiredCondition.class)
@Target(ElementType.TYPE)
public @interface RequiresDocker {
}