honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Work on migrator

This commit is contained in:
James Agnew 2018-09-04 10:32:08 +08:00
parent 0f7c4219cb
commit 77305eb570
45 changed files with 1223 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceEncodingEnum.java
View File

@ -34,10 +34,14 @@ public enum ResourceEncodingEnum {
* See ResourceHistoryTable RES_ENCODING column
*/
/** Json */
/**
* Json
*/
JSON,
/** Json Compressed */
/**
* Json Compressed
*/
JSONC,
/**

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamCoords.java
View File

@ -32,7 +32,7 @@ import javax.persistence.*;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_COORDS", indexes = {
@Index(name = "IDX_SP_COORDS_HASH", columnList = "HASH_IDENTITY,SP_VALUE,SP_LATITUDE,SP_LONGITUDE"),
@Index(name = "IDX_SP_COORDS_HASH", columnList = "HASH_IDENTITY,SP_LATITUDE,SP_LONGITUDE"),
@Index(name = "IDX_SP_COORDS_UPDATED", columnList = "SP_UPDATED"),
@Index(name = "IDX_SP_COORDS_RESID", columnList = "RES_ID")
})

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamDate.java
View File

@ -37,7 +37,7 @@ import java.util.Date;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_DATE", indexes = {
// @Index(name = "IDX_SP_DATE", columnList = "RES_TYPE,SP_NAME,SP_VALUE_LOW,SP_VALUE_HIGH"),
// We previously had an index called IDX_SP_DATE - Dont reuse
@Index(name = "IDX_SP_DATE_HASH", columnList = "HASH_IDENTITY,SP_VALUE_LOW,SP_VALUE_HIGH"),
@Index(name = "IDX_SP_DATE_UPDATED", columnList = "SP_UPDATED"),
@Index(name = "IDX_SP_DATE_RESID", columnList = "RES_ID")

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamNumber.java
View File

@ -37,7 +37,7 @@ import java.math.BigDecimal;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_NUMBER", indexes = {
// @Index(name = "IDX_SP_NUMBER", columnList = "RES_TYPE,SP_NAME,SP_VALUE"),
// We used to have an index with name IDX_SP_NUMBER - Dont reuse
@Index(name = "IDX_SP_NUMBER_HASH_VAL", columnList = "HASH_IDENTITY,SP_VALUE"),
@Index(name = "IDX_SP_NUMBER_UPDATED", columnList = "SP_UPDATED"),
@Index(name = "IDX_SP_NUMBER_RESID", columnList = "RES_ID")

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamQuantity.java
View File

@ -38,9 +38,10 @@ import java.math.BigDecimal;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_QUANTITY", indexes = {
// @Index(name = "IDX_SP_QUANTITY", columnList = "RES_TYPE,SP_NAME,SP_SYSTEM,SP_UNITS,SP_VALUE"),
// We used to have an index named IDX_SP_QUANTITY - Dont reuse
@Index(name = "IDX_SP_QUANTITY_HASH", columnList = "HASH_IDENTITY,SP_VALUE"),
@Index(name = "IDX_SP_QUANTITY_HASH_UN", columnList = "HASH_IDENTITY_AND_UNITS,SP_VALUE"),
@Index(name = "IDX_SP_QUANTITY_HASH_SYSUN", columnList = "HASH_IDENTITY_SYS_UNITS,SP_VALUE"),
@Index(name = "IDX_SP_QUANTITY_UPDATED", columnList = "SP_UPDATED"),
@Index(name = "IDX_SP_QUANTITY_RESID", columnList = "RES_ID")
})

45
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceLink.java
View File

@ -20,36 +20,21 @@ package ca.uhn.fhir.jpa.entity;
* #L%
*/
import java.io.Serializable;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.ForeignKey;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.hibernate.annotations.ColumnDefault;
import org.hibernate.search.annotations.Field;
import org.hl7.fhir.instance.model.api.IIdType;
import javax.persistence.*;
import java.io.Serializable;
import java.util.Date;
@Entity
@Table(name = "HFJ_RES_LINK" , indexes= {
@Index(name="IDX_RL_TPATHRES", columnList= "SRC_PATH,TARGET_RESOURCE_ID"),
@Index(name="IDX_RL_SRC", columnList= "SRC_RESOURCE_ID"),
@Index(name="IDX_RL_DEST", columnList= "TARGET_RESOURCE_ID")
@Table(name = "HFJ_RES_LINK", indexes = {
@Index(name = "IDX_RL_TPATHRES", columnList = "SRC_PATH,TARGET_RESOURCE_ID"),
@Index(name = "IDX_RL_SRC", columnList = "SRC_RESOURCE_ID"),
@Index(name = "IDX_RL_DEST", columnList = "TARGET_RESOURCE_ID")
})
public class ResourceLink implements Serializable {
@ -64,30 +49,30 @@ public class ResourceLink implements Serializable {
@Column(name = "SRC_PATH", length = 100, nullable = false)
private String mySourcePath;
@ManyToOne(optional = false, fetch=FetchType.LAZY)
@JoinColumn(name = "SRC_RESOURCE_ID", referencedColumnName = "RES_ID", nullable = false, foreignKey=@ForeignKey(name="FK_RESLINK_SOURCE"))
@ManyToOne(optional = false, fetch = FetchType.LAZY)
@JoinColumn(name = "SRC_RESOURCE_ID", referencedColumnName = "RES_ID", nullable = false, foreignKey = @ForeignKey(name = "FK_RESLINK_SOURCE"))
private ResourceTable mySourceResource;
@Column(name = "SRC_RESOURCE_ID", insertable = false, updatable = false, nullable = false)
private Long mySourceResourcePid;
@Column(name = "SOURCE_RESOURCE_TYPE", nullable=false, length=ResourceTable.RESTYPE_LEN)
@Column(name = "SOURCE_RESOURCE_TYPE", nullable = false, length = ResourceTable.RESTYPE_LEN)
@Field()
private String mySourceResourceType;
@ManyToOne(optional = true, fetch=FetchType.LAZY)
@JoinColumn(name = "TARGET_RESOURCE_ID", referencedColumnName = "RES_ID", nullable = true, foreignKey=@ForeignKey(name="FK_RESLINK_TARGET"))
@ManyToOne(optional = true, fetch = FetchType.LAZY)
@JoinColumn(name = "TARGET_RESOURCE_ID", referencedColumnName = "RES_ID", nullable = true, foreignKey = @ForeignKey(name = "FK_RESLINK_TARGET"))
private ResourceTable myTargetResource;
@Column(name = "TARGET_RESOURCE_ID", insertable = false, updatable = false, nullable = true)
@Field()
private Long myTargetResourcePid;
@Column(name = "TARGET_RESOURCE_TYPE", nullable=false, length=ResourceTable.RESTYPE_LEN)
@Column(name = "TARGET_RESOURCE_TYPE", nullable = false, length = ResourceTable.RESTYPE_LEN)
@Field()
private String myTargetResourceType;
@Column(name = "TARGET_RESOURCE_URL", length=200, nullable = true)
@Column(name = "TARGET_RESOURCE_URL", length = 200, nullable = true)
@Field()
private String myTargetResourceUrl;

20
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceSearchView.java
View File

@ -19,25 +19,17 @@ package ca.uhn.fhir.jpa.entity;
* limitations under the License.
* #L%
*/
import java.io.Serializable;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Id;
import javax.persistence.Lob;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.hibernate.annotations.Immutable;
import org.hibernate.annotations.Subselect;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.InstantDt;
import ca.uhn.fhir.rest.api.Constants;
import org.hibernate.annotations.Immutable;
import org.hibernate.annotations.Subselect;
import javax.persistence.*;
import java.io.Serializable;
import java.util.Date;
//@formatter:off
@Entity

1
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/SearchParamPresent.java
View File

@ -28,6 +28,7 @@ import java.io.Serializable;
@Entity
@Table(name = "HFJ_RES_PARAM_PRESENT", indexes = {
// We used to have a constraint named IDX_RESPARMPRESENT_SPID_RESID - Don't reuse
@Index(name = "IDX_RESPARMPRESENT_RESID", columnList = "RES_ID"),
@Index(name = "IDX_RESPARMPRESENT_HASHPRES", columnList = "HASH_PRESENCE")
})

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java
View File

@ -93,15 +93,17 @@ public class TermConcept implements Serializable {
private Long myIndexStatus;
@Field(name = "myParentPids", index = org.hibernate.search.annotations.Index.YES, store = Store.YES, analyze = Analyze.YES, analyzer = @Analyzer(definition = "conceptParentPidsAnalyzer"))
@Lob
@Column(name="PARENT_PIDS", nullable = true)
@Column(name = "PARENT_PIDS", nullable = true)
private String myParentPids;
@OneToMany(cascade = {}, fetch = FetchType.LAZY, mappedBy = "myChild")
private Collection<TermConceptParentChildLink> myParents;
@Column(name = "CODE_SEQUENCE", nullable = true)
private Integer mySequence;
public TermConcept() {
super();
}
public TermConcept(TermCodeSystemVersion theCs, String theCode) {
setCodeSystemVersion(theCs);
setCode(theCode);

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptMapGroup.java
View File

@ -38,7 +38,7 @@ public class TermConceptMapGroup implements Serializable {
private Long myId;
@ManyToOne()
@JoinColumn(name = "CONCEPT_MAP_PID", nullable = false, referencedColumnName = "PID", foreignKey=@ForeignKey(name="FK_TCMGROUP_CONCEPTMAP"))
@JoinColumn(name = "CONCEPT_MAP_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TCMGROUP_CONCEPTMAP"))
private TermConceptMap myConceptMap;
@Column(name = "SOURCE_URL", nullable = false, length = 200)

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptMapGroupElement.java
View File

@ -42,7 +42,7 @@ public class TermConceptMapGroupElement implements Serializable {
private Long myId;
@ManyToOne()
@JoinColumn(name = "CONCEPT_MAP_GROUP_PID", nullable = false, referencedColumnName = "PID", foreignKey=@ForeignKey(name="FK_TCMGELEMENT_GROUP"))
@JoinColumn(name = "CONCEPT_MAP_GROUP_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TCMGELEMENT_GROUP"))
private TermConceptMapGroup myConceptMapGroup;
@Column(name = "SOURCE_CODE", nullable = false, length = TermConcept.CODE_LENGTH)

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConceptMapGroupElementTarget.java
View File

@ -41,7 +41,7 @@ public class TermConceptMapGroupElementTarget implements Serializable {
private Long myId;
@ManyToOne()
@JoinColumn(name = "CONCEPT_MAP_GRP_ELM_PID", nullable = false, referencedColumnName = "PID", foreignKey=@ForeignKey(name="FK_TCMGETARGET_ELEMENT"))
@JoinColumn(name = "CONCEPT_MAP_GRP_ELM_PID", nullable = false, referencedColumnName = "PID", foreignKey = @ForeignKey(name = "FK_TCMGETARGET_ELEMENT"))
private TermConceptMapGroupElement myConceptMapGroupElement;
@Column(name = "TARGET_CODE", nullable = false, length = TermConcept.CODE_LENGTH)

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java
View File

@ -105,7 +105,7 @@ public class TestR4Config extends BaseJavaConfigR4 {
DataSource dataSource = ProxyDataSourceBuilder
.create(retVal)
// .logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
.logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
.logSlowQueryBySlf4j(10, TimeUnit.SECONDS)
.countQuery(new ThreadQueryCountHolder())
.build();

107
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java
View File

@ -1,17 +1,22 @@
package ca.uhn.fhir.jpa.migrate;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTableColumnTypeTask;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.ColumnMapRowMapper;
import org.thymeleaf.util.StringUtils;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.*;
import java.util.HashSet;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
public class JdbcUtils {
private static final Logger ourLog = LoggerFactory.getLogger(JdbcUtils.class);
/**
* Retrieve all index names
*/
@ -19,14 +24,18 @@ public class JdbcUtils {
DataSource dataSource = Objects.requireNonNull(theConnectionProperties.getDataSource());
Connection connection = dataSource.getConnection();
return theConnectionProperties.getTxTemplate().execute(t -> {
DatabaseMetaData metadata = null;
DatabaseMetaData metadata;
try {
metadata = connection.getMetaData();
ResultSet indexes = metadata.getIndexInfo(null, null, theTableName, false, false);
Set<String> indexNames = new HashSet<>();
while (indexes.next()) {
ourLog.debug("*** Next index: {}", new ColumnMapRowMapper().mapRow(indexes, 0));
String indexName = indexes.getString("INDEX_NAME");
indexName = StringUtils.toUpperCase(indexName, Locale.US);
indexNames.add(indexName);
}
@ -37,4 +46,92 @@ public class JdbcUtils {
});
}
@SuppressWarnings("ConstantConditions")
public static boolean isIndexUnique(DriverTypeEnum.ConnectionProperties theConnectionProperties, String theTableName, String theIndexName) throws SQLException {
DataSource dataSource = Objects.requireNonNull(theConnectionProperties.getDataSource());
Connection connection = dataSource.getConnection();
return theConnectionProperties.getTxTemplate().execute(t -> {
DatabaseMetaData metadata;
try {
metadata = connection.getMetaData();
ResultSet indexes = metadata.getIndexInfo(null, null, theTableName, false, false);
while (indexes.next()) {
String indexName = indexes.getString("INDEX_NAME");
if (indexName.equalsIgnoreCase(theIndexName)) {
boolean nonUnique = indexes.getBoolean("NON_UNIQUE");
return !nonUnique;
}
}
} catch (SQLException e) {
throw new InternalErrorException(e);
}
throw new InternalErrorException("Can't find index: " + theIndexName + " on table " + theTableName);
});
}
/**
* Retrieve all index names
*/
public static String getColumnType(DriverTypeEnum.ConnectionProperties theConnectionProperties, String theTableName, String theColumnName) throws SQLException {
DataSource dataSource = Objects.requireNonNull(theConnectionProperties.getDataSource());
try (Connection connection = dataSource.getConnection()) {
return theConnectionProperties.getTxTemplate().execute(t -> {
DatabaseMetaData metadata;
try {
metadata = connection.getMetaData();
ResultSet indexes = metadata.getColumns(null, null, theTableName, theColumnName);
indexes.next();
int dataType = indexes.getInt("DATA_TYPE");
Long length = indexes.getLong("COLUMN_SIZE");
switch (dataType) {
case Types.VARCHAR:
return BaseTableColumnTypeTask.ColumnTypeEnum.STRING.getDescriptor(length);
case Types.BIGINT:
return BaseTableColumnTypeTask.ColumnTypeEnum.LONG.getDescriptor(length);
default:
throw new IllegalArgumentException("Don't know how to handle datatype: " + dataType);
}
} catch (SQLException e) {
throw new InternalErrorException(e);
}
});
}
}
/**
* Retrieve all index names
*/
public static Set<String> getColumnNames(DriverTypeEnum.ConnectionProperties theConnectionProperties, String theTableName) throws SQLException {
DataSource dataSource = Objects.requireNonNull(theConnectionProperties.getDataSource());
Connection connection = dataSource.getConnection();
return theConnectionProperties.getTxTemplate().execute(t -> {
DatabaseMetaData metadata;
try {
metadata = connection.getMetaData();
ResultSet indexes = metadata.getColumns(null, null, theTableName, null);
Set<String> columnNames = new HashSet<>();
while (indexes.next()) {
String columnName = indexes.getString("COLUMN_NAME");
columnName = StringUtils.toUpperCase(columnName, Locale.US);
columnNames.add(columnName);
}
return columnNames;
} catch (SQLException e) {
throw new InternalErrorException(e);
}
});
}
}

30
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java Normal file
View File

@ -0,0 +1,30 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.migrate.JdbcUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.util.Set;
public class AddColumnTask extends BaseTableColumnTypeTask<AddColumnTask> {
private static final Logger ourLog = LoggerFactory.getLogger(AddColumnTask.class);
@Override
public void execute() throws SQLException {
Set<String> columnNames = JdbcUtils.getColumnNames(getConnectionProperties(), getTableName());
if (columnNames.contains(getColumnName())) {
ourLog.info("Column {} already exists on table {} - No action performed", getColumnName(), getTableName());
return;
}
String type = getSqlType();
String nullable = getSqlNotNull();
String sql = "alter table " + getTableName() + " add column " + getColumnName() + " " + type + " " + nullable;
ourLog.info("Adding column {} of type {} to table {}", getColumnName(), type, getTableName());
executeSql(sql);
}
}

19
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddIndexTask.java
View File

@ -4,26 +4,23 @@ import ca.uhn.fhir.jpa.migrate.JdbcUtils;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.thymeleaf.util.StringUtils;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Set;
public class AddIndexTask extends BaseTask<AddIndexTask> {
public class AddIndexTask extends BaseTableTask<AddIndexTask> {
private static final Logger ourLog = LoggerFactory.getLogger(AddIndexTask.class);
private String myTableName;
private String myIndexName;
private List<String> myColumns;
private Boolean myUnique;
public void setTableName(String theTableName) {
myTableName = theTableName;
}
public void setIndexName(String theIndexName) {
myIndexName = theIndexName;
myIndexName = StringUtils.toUpperCase(theIndexName, Locale.US);
}
public void setColumns(List<String> theColumns) {
@ -36,23 +33,23 @@ public class AddIndexTask extends BaseTask<AddIndexTask> {
@Override
public void validate() {
super.validate();
Validate.notBlank(myIndexName, "Index name not specified");
Validate.notBlank(myTableName, "Table name not specified");
Validate.isTrue(myColumns.size() > 0, "Columns not specified");
Validate.notNull(myUnique, "Uniqueness not specified");
}
@Override
public void execute() throws SQLException {
Set<String> indexNames = JdbcUtils.getIndexNames(getConnectionProperties(), myTableName);
Set<String> indexNames = JdbcUtils.getIndexNames(getConnectionProperties(), getTableName());
if (indexNames.contains(myIndexName)) {
ourLog.info("Index {} already exists on table {} - No action performed", myIndexName, myTableName);
ourLog.info("Index {} already exists on table {} - No action performed", myIndexName, getTableName());
return;
}
String unique = myUnique ? "UNIQUE " : "";
String columns = String.join(", ", myColumns);
String sql = "CREATE " + unique + " INDEX " + myIndexName + " ON " + myTableName + "(" + columns + ")";
String sql = "CREATE " + unique + " INDEX " + myIndexName + " ON " + getTableName() + "(" + columns + ")";
executeSql(sql);
}

90
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ArbitrarySqlTask.java Normal file
View File

@ -0,0 +1,90 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.ColumnMapRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
public class ArbitrarySqlTask extends BaseTask<ArbitrarySqlTask> {
private static final Logger ourLog = LoggerFactory.getLogger(ArbitrarySqlTask.class);
private final String myDescription;
private List<Task> myTask = new ArrayList<>();
private int myBatchSize = 1000;
public ArbitrarySqlTask(String theDescription) {
myDescription = theDescription;
}
public void addQuery(String theSql, QueryModeEnum theMode, Consumer<Map<String, Object>> theConsumer) {
myTask.add(new QueryTask(theSql, theMode, theConsumer));
}
@Override
public void validate() {
// nothing
}
@Override
public void execute() throws SQLException {
ourLog.info("Starting: {}", myDescription);
for (Task next : myTask) {
next.execute();
}
}
public void setBatchSize(int theBatchSize) {
myBatchSize = theBatchSize;
}
public enum QueryModeEnum {
BATCH_UNTIL_NO_MORE
}
private abstract class Task {
public abstract void execute();
}
private class QueryTask extends Task {
private final String mySql;
private final QueryModeEnum myMode;
private final Consumer<Map<String, Object>> myConsumer;
public QueryTask(String theSql, QueryModeEnum theMode, Consumer<Map<String, Object>> theConsumer) {
mySql = theSql;
myMode = theMode;
myConsumer = theConsumer;
}
@Override
public void execute() {
List<Map<String, Object>> rows;
do {
ourLog.info("Querying for up to {} rows", myBatchSize);
rows = getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = newJdbcTemnplate();
jdbcTemplate.setMaxRows(myBatchSize);
return jdbcTemplate.query(mySql, new ColumnMapRowMapper());
});
ourLog.info("Processing {} rows", rows.size());
List<Map<String, Object>> finalRows = rows;
getTxTemplate().execute(t -> {
for (Map<String, Object> nextRow : finalRows) {
myConsumer.accept(nextRow);
}
return null;
});
} while (rows.size() > 0);
}
}
}

30
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTableColumnTask.java Normal file
View File

@ -0,0 +1,30 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import org.apache.commons.lang3.Validate;
import org.thymeleaf.util.StringUtils;
import java.util.Locale;
public abstract class BaseTableColumnTask<T extends BaseTableTask> extends BaseTableTask<T> {
private String myColumnName;
@SuppressWarnings("unchecked")
public T setColumnName(String theColumnName) {
myColumnName = StringUtils.toUpperCase(theColumnName, Locale.US);
return (T) this;
}
public String getColumnName() {
return myColumnName;
}
@Override
public void validate() {
super.validate();
Validate.notBlank(myColumnName, "Column name not specified");
}
}

122
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTableColumnTypeTask.java Normal file
View File

@ -0,0 +1,122 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.migrate.DriverTypeEnum;
import org.apache.commons.lang3.Validate;
import org.springframework.util.Assert;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
public abstract class BaseTableColumnTypeTask<T extends BaseTableTask> extends BaseTableColumnTask<T> {
private ColumnTypeEnum myColumnType;
private Map<ColumnTypeEnum, Map<DriverTypeEnum, String>> myColumnTypeToDriverTypeToSqlType = new HashMap<>();
private Boolean myNullable;
private Long myColumnLength;
/**
* Constructor
*/
BaseTableColumnTypeTask() {
setColumnType(ColumnTypeEnum.LONG, DriverTypeEnum.DERBY_EMBEDDED, "bigint");
setColumnType(ColumnTypeEnum.LONG, DriverTypeEnum.MARIADB_10_1, "bigint");
setColumnType(ColumnTypeEnum.LONG, DriverTypeEnum.MYSQL_5_7, "bigint");
setColumnType(ColumnTypeEnum.LONG, DriverTypeEnum.MSSQL_2012, "bigint");
setColumnType(ColumnTypeEnum.LONG, DriverTypeEnum.ORACLE_12C, "number(19,0)");
setColumnType(ColumnTypeEnum.LONG, DriverTypeEnum.POSTGRES_9_4, "int8");
setColumnType(ColumnTypeEnum.STRING, DriverTypeEnum.DERBY_EMBEDDED, "varchar(?)");
setColumnType(ColumnTypeEnum.STRING, DriverTypeEnum.MARIADB_10_1, "varchar(?)");
setColumnType(ColumnTypeEnum.STRING, DriverTypeEnum.MYSQL_5_7, "varchar(?)");
setColumnType(ColumnTypeEnum.STRING, DriverTypeEnum.MSSQL_2012, "varchar(?)");
setColumnType(ColumnTypeEnum.STRING, DriverTypeEnum.ORACLE_12C, "varchar2(?)");
setColumnType(ColumnTypeEnum.STRING, DriverTypeEnum.POSTGRES_9_4, "varchar(?)");
}
public ColumnTypeEnum getColumnType() {
return myColumnType;
}
private void setColumnType(ColumnTypeEnum theColumnType, DriverTypeEnum theDriverType, String theColumnTypeSql) {
Map<DriverTypeEnum, String> columnSqlType = myColumnTypeToDriverTypeToSqlType.computeIfAbsent(theColumnType, k -> new HashMap<>());
if (columnSqlType.containsKey(theDriverType)) {
throw new IllegalStateException("Duplicate key: " + theDriverType);
}
columnSqlType.put(theDriverType, theColumnTypeSql);
}
@Override
public void validate() {
super.validate();
Validate.notNull(myColumnType);
Validate.notNull(myNullable);
if (myColumnType == ColumnTypeEnum.STRING) {
Validate.notNull(myColumnLength);
} else {
Validate.isTrue(myColumnLength == null);
}
}
@SuppressWarnings("unchecked")
public T setColumnType(ColumnTypeEnum theColumnType) {
myColumnType = theColumnType;
return (T) this;
}
protected String getSqlType() {
String retVal = myColumnTypeToDriverTypeToSqlType.get(myColumnType).get(getDriverType());
Objects.requireNonNull(retVal);
if (myColumnType == ColumnTypeEnum.STRING) {
retVal = retVal.replace("?", Long.toString(getColumnLength()));
}
return retVal;
}
public boolean isNullable() {
return myNullable;
}
public void setNullable(boolean theNullable) {
myNullable = theNullable;
}
protected String getSqlNotNull() {
return isNullable() ? "" : " not null";
}
public Long getColumnLength() {
return myColumnLength;
}
public void setColumnLength(int theColumnLength) {
myColumnLength = (long) theColumnLength;
}
public enum ColumnTypeEnum {
LONG {
@Override
public String getDescriptor(Long theColumnLength) {
Assert.isTrue(theColumnLength == null, "Must not supply a column length");
return "bigint";
}
},
STRING {
@Override
public String getDescriptor(Long theColumnLength) {
Assert.isTrue(theColumnLength != null, "Must supply a column length");
return "varchar(" + theColumnLength + ")";
}
};
public abstract String getDescriptor(Long theColumnLength);
}
}

21
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTableTask.java Normal file
View File

@ -0,0 +1,21 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import org.apache.commons.lang3.Validate;
public abstract class BaseTableTask<T extends BaseTableTask> extends BaseTask {
private String myTableName;
public String getTableName() {
return myTableName;
}
public T setTableName(String theTableName) {
myTableName = theTableName;
return (T) this;
}
@Override
public void validate() {
Validate.notBlank(myTableName);
}
}

2
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTask.java
View File

@ -34,7 +34,7 @@ public abstract class BaseTask<T extends BaseTask> {
Integer changes = getConnectionProperties().getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = getConnectionProperties().newJdbcTemplate();
int changesCount = jdbcTemplate.update(theSql, theArguments);
ourLog.info("SQL {} returned {}", theSql, changesCount);
ourLog.info("SQL \"{}\" returned {}", theSql, changesCount);
return changesCount;
});

130
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/CalculateHashesTask.java
View File

@ -1,57 +1,129 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.util.StopWatch;
import com.google.common.collect.ForwardingMap;
import org.apache.commons.lang3.Validate;
import org.checkerframework.checker.nullness.compatqual.NullableDecl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.springframework.jdbc.core.RowMapper;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
public class CalculateHashesTask extends BaseTask {
public class CalculateHashesTask extends BaseTableColumnTask<CalculateHashesTask> {
private String myTableName;
private String myColumnName;
private static final Logger ourLog = LoggerFactory.getLogger(CalculateHashesTask.class);
private int myBatchSize = 10000;
private Map<String, Function<MandatoryKeyMap<String, Object>, Long>> myCalculators = new HashMap<>();
public void setTableName(String theTableName) {
myTableName = theTableName;
public void setBatchSize(int theBatchSize) {
myBatchSize = theBatchSize;
}
public void setColumnName(String theColumnName) {
myColumnName = theColumnName;
}
@Override
public void validate() {
Validate.notBlank(myTableName);
Validate.notBlank(myColumnName);
}
@Override
public void execute() {
List<Map<String, Object>> rows = getTxTemplate().execute(t->{
List<Map<String, Object>> rows;
do {
rows = getTxTemplate().execute(t -> {
JdbcTemplate jdbcTemplate = newJdbcTemnplate();
int batchSize = 10000;
jdbcTemplate.setMaxRows(batchSize);
String sql = "SELECT * FROM " + myTableName + " WHERE " + myColumnName + " IS NULL";
ourLog.info("Loading up to {} rows in {} with no hashes", batchSize, myTableName);
jdbcTemplate.setMaxRows(myBatchSize);
String sql = "SELECT * FROM " + getTableName() + " WHERE " + getColumnName() + " IS NULL";
ourLog.info("Finding up to {} rows in {} that requires hashes", myBatchSize, getTableName());
return jdbcTemplate.queryForList(sql);
});
updateRows(rows);
} while (rows.size() > 0);
}
private void updateRows(List<Map<String, Object>> theRows) {
StopWatch sw = new StopWatch();
getTxTemplate().execute(t -> {
// Loop through rows
assert theRows != null;
for (Map<String, Object> nextRow : theRows) {
Map<String, Long> newValues = new HashMap<>();
MandatoryKeyMap<String, Object> nextRowMandatoryKeyMap = new MandatoryKeyMap<>(nextRow);
// Apply calculators
for (Map.Entry<String, Function<MandatoryKeyMap<String, Object>, Long>> nextCalculatorEntry : myCalculators.entrySet()) {
String nextColumn = nextCalculatorEntry.getKey();
Function<MandatoryKeyMap<String, Object>, Long> nextCalculator = nextCalculatorEntry.getValue();
Long value = nextCalculator.apply(nextRowMandatoryKeyMap);
newValues.put(nextColumn, value);
}
// Generate update SQL
StringBuilder sqlBuilder = new StringBuilder();
List<Long> arguments = new ArrayList<>();
sqlBuilder.append("UPDATE ");
sqlBuilder.append(getTableName());
sqlBuilder.append(" SET ");
for (Map.Entry<String, Long> nextNewValueEntry : newValues.entrySet()) {
if (arguments.size() > 0) {
sqlBuilder.append(", ");
}
sqlBuilder.append(nextNewValueEntry.getKey()).append(" = ?");
arguments.add(nextNewValueEntry.getValue());
}
sqlBuilder.append(" WHERE SP_ID = ?");
arguments.add((Long) nextRow.get("SP_ID"));
// Apply update SQL
newJdbcTemnplate().update(sqlBuilder.toString(), arguments.toArray());
}
private Map<String, Function<Map<String, Object>, Long>> myColumnMappers;
public void addCalculator(String theColumnName, Function<Map<String, Object>, Long> theConsumer) {
return theRows.size();
});
ourLog.info("Updated {} rows on {} in {}", theRows.size(), getTableName(), sw.toString());
}
private static final Logger ourLog = LoggerFactory.getLogger(CalculateHashesTask.class);
public CalculateHashesTask addCalculator(String theColumnName, Function<MandatoryKeyMap<String, Object>, Long> theConsumer) {
Validate.isTrue(myCalculators.containsKey(theColumnName) == false);
myCalculators.put(theColumnName, theConsumer);
return this;
}
public static class MandatoryKeyMap<K, V> extends ForwardingMap<K, V> {
private final Map<K, V> myWrap;
public MandatoryKeyMap(Map<K, V> theWrap) {
myWrap = theWrap;
}
@Override
public V get(@NullableDecl Object theKey) {
if (!containsKey(theKey)) {
throw new IllegalArgumentException("No key: " + theKey);
}
return super.get(theKey);
}
public String getString(String theKey) {
return (String) get(theKey);
}
@Override
protected Map<K, V> delegate() {
return myWrap;
}
public String getResourceType() {
return getString("RES_TYPE");
}
public String getParamName() {
return getString("SP_NAME");
}
}
}

44
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java
View File

@ -8,38 +8,58 @@ import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.util.Set;
public class DropIndexTask extends BaseTask<DropIndexTask> {
public class DropIndexTask extends BaseTableTask<DropIndexTask> {
private static final Logger ourLog = LoggerFactory.getLogger(DropIndexTask.class);
private String myIndexName;
private String myTableName;
@Override
public void validate() {
super.validate();
Validate.notBlank(myIndexName, "The index name must not be blank");
Validate.notBlank(myTableName, "The table name must not be blank");
if (getDescription() == null) {
setDescription("Drop index " + myIndexName + " on table " + myTableName);
setDescription("Drop index " + myIndexName + " on table " + getTableName());
}
}
@Override
public void execute() throws SQLException {
Set<String> indexNames = JdbcUtils.getIndexNames(getConnectionProperties(), myTableName);
Set<String> indexNames = JdbcUtils.getIndexNames(getConnectionProperties(), getTableName());
if (!indexNames.contains(myIndexName)) {
ourLog.info("Index {} does not exist on table {} - No action needed", myIndexName, myTableName);
ourLog.info("Index {} does not exist on table {} - No action needed", myIndexName, getTableName());
return;
}
ourLog.info("Dropping index {} on table {}", myIndexName, myTableName);
boolean isUnique = JdbcUtils.isIndexUnique(getConnectionProperties(), getTableName(), myIndexName);
String uniquenessString = isUnique ? "unique" : "non-unique";
ourLog.info("Dropping {} index {} on table {}", uniquenessString, myIndexName, getTableName());
String sql = null;
if (isUnique) {
// Drop constraint
switch (getDriverType()) {
case MYSQL_5_7:
case MARIADB_10_1:
sql = "ALTER TABLE " + myTableName + " DROP INDEX " + myIndexName;
sql = "ALTER TABLE " + getTableName() + " DROP INDEX " + myIndexName;
break;
case DERBY_EMBEDDED:
sql = "DROP INDEX " + myIndexName;
break;
case POSTGRES_9_4:
case ORACLE_12C:
case MSSQL_2012:
sql = "ALTER TABLE " + getTableName() + " DROP CONSTRAINT " + myIndexName;
break;
}
} else {
// Drop index
switch (getDriverType()) {
case MYSQL_5_7:
case MARIADB_10_1:
sql = "ALTER TABLE " + getTableName() + " DROP INDEX " + myIndexName;
break;
case POSTGRES_9_4:
case DERBY_EMBEDDED:
@ -47,19 +67,15 @@ public class DropIndexTask extends BaseTask<DropIndexTask> {
sql = "DROP INDEX " + myIndexName;
break;
case MSSQL_2012:
sql = "DROP INDEX " + myTableName + "." + myIndexName;
sql = "DROP INDEX " + getTableName() + "." + myIndexName;
break;
}
}
executeSql(sql);
}
public DropIndexTask setTableName(String theTableName) {
myTableName = theTableName;
return this;
}
public DropIndexTask setIndexName(String theIndexName) {
myIndexName = theIndexName;
return this;

69
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ModifyColumnTask.java Normal file
View File

@ -0,0 +1,69 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.migrate.JdbcUtils;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
public class ModifyColumnTask extends BaseTableColumnTypeTask<ModifyColumnTask> {
private static final Logger ourLog = LoggerFactory.getLogger(ModifyColumnTask.class);
@Override
public void execute() {
String existingType;
try {
existingType = JdbcUtils.getColumnType(getConnectionProperties(), getTableName(), getColumnName());
} catch (SQLException e) {
throw new InternalErrorException(e);
}
String wantedType = getColumnType().getDescriptor(getColumnLength());
if (existingType.equals(wantedType)) {
ourLog.info("Column {} on table {} is already of type {} - No action performed", getColumnName(), getTableName(), wantedType);
return;
}
String type = getSqlType();
String notNull = getSqlNotNull();
String sql;
String sqlNotNull = null;
switch (getDriverType()) {
case DERBY_EMBEDDED:
sql = "alter table " + getTableName() + " alter column " + getColumnName() + " set data type " + type;
break;
case MARIADB_10_1:
case MYSQL_5_7:
sql = "alter table " + getTableName() + " modify column " + getColumnName() + " " + type + notNull;
break;
case POSTGRES_9_4:
sql = "alter table " + getTableName() + " alter column " + getColumnName() + " type " + type;
if (isNullable() == false) {
sqlNotNull = "alter table " + getTableName() + " alter column " + getColumnName() + " set not null";
}
break;
case ORACLE_12C:
sql = "alter table " + getTableName() + " modify " + getColumnName() + " " + type + notNull;
break;
case MSSQL_2012:
sql = "alter table " + getTableName() + " alter column " + getColumnName() + " " + type + notNull;
break;
default:
throw new IllegalStateException("Dont know how to handle " + getDriverType());
}
ourLog.info("Updating column {} on table {} to type {}", getColumnName(), getTableName(), type);
executeSql(sql);
if (sqlNotNull != null) {
ourLog.info("Updating column {} on table {} to not null", getColumnName(), getTableName());
executeSql(sqlNotNull);
}
}
}

312
hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java
View File

@ -1,10 +1,8 @@
package ca.uhn.fhir.jpa.migrate.tasks;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamCoords;
import ca.uhn.fhir.jpa.migrate.taskdef.AddIndexTask;
import ca.uhn.fhir.jpa.migrate.taskdef.BaseTask;
import ca.uhn.fhir.jpa.migrate.taskdef.CalculateHashesTask;
import ca.uhn.fhir.jpa.migrate.taskdef.DropIndexTask;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.migrate.taskdef.*;
import ca.uhn.fhir.util.VersionEnum;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
@ -20,31 +18,240 @@ public class HapiFhirJpaMigrationTasks {
public HapiFhirJpaMigrationTasks() {
// Forced ID changes
forVersion(VersionEnum.V3_5_0)
.onTable("HFJ_FORCED_ID")
Builder.BuilderWithTableName forcedId = forVersion(VersionEnum.V3_5_0).onTable("HFJ_FORCED_ID");
forcedId
.dropIndex("IDX_FORCEDID_TYPE_FORCEDID");
forVersion(VersionEnum.V3_5_0)
.onTable("HFJ_FORCED_ID")
forcedId
.dropIndex("IDX_FORCEDID_TYPE_RESID");
forVersion(VersionEnum.V3_5_0)
.onTable("HFJ_FORCED_ID")
forcedId
.addIndex("IDX_FORCEDID_TYPE_FID")
.unique(true)
.withColumns("RESOURCE_TYPE", "FORCED_ID");
// Indexes - Coords
forVersion(VersionEnum.V3_5_0)
.onTable("HFJ_SPIDX_COORDS")
Builder.BuilderWithTableName spidxCoords = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_COORDS");
spidxCoords
.dropIndex("IDX_SP_COORDS_HASH");
forVersion(VersionEnum.V3_5_0)
.onTable("HFJ_SPIDX_COORDS")
spidxCoords
.addColumn("HASH_IDENTITY")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxCoords
.addIndex("IDX_SP_COORDS_HASH")
.unique(false)
.withColumns("HASH_IDENTITY", "SP_VALUE", "SP_LATITUDE", "SP_LONGITUDE");
forVersion(VersionEnum.V3_5_0)
.addTask(new CalculateHashesTask().calculator(()->{
return ResourceIndexedSearchParamCoords.calculateHashIdentity("resourceType", "paramName");
}));
spidxCoords
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")))
);
// Indexes - Date
Builder.BuilderWithTableName spidxDate = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_DATE");
spidxDate
.dropIndex("IDX_SP_TOKEN");
spidxDate
.addColumn("HASH_IDENTITY")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxDate
.addIndex("IDX_SP_DATE_HASH")
.unique(false)
.withColumns("HASH_IDENTITY", "SP_VALUE_LOW", "SP_VALUE_HIGH");
spidxDate
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")))
);
// Indexes - Number
Builder.BuilderWithTableName spidxNumber = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_NUMBER");
spidxNumber
.dropIndex("IDX_SP_NUMBER");
spidxNumber
.addColumn("HASH_IDENTITY")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxNumber
.addIndex("IDX_SP_NUMBER_HASH_VAL")
.unique(false)
.withColumns("HASH_IDENTITY", "SP_VALUE");
spidxNumber
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")))
);
// Indexes - Quantity
Builder.BuilderWithTableName spidxQuantity = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_QUANTITY");
spidxQuantity
.dropIndex("IDX_SP_QUANTITY");
spidxQuantity
.addColumn("HASH_IDENTITY")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxQuantity
.addColumn("HASH_IDENTITY_SYS_UNITS")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxQuantity
.addColumn("HASH_IDENTITY_AND_UNITS")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxQuantity
.addIndex("IDX_SP_QUANTITY_HASH")
.unique(false)
.withColumns("HASH_IDENTITY", "SP_VALUE");
spidxQuantity
.addIndex("IDX_SP_QUANTITY_HASH_UN")
.unique(false)
.withColumns("HASH_IDENTITY_AND_UNITS", "SP_VALUE");
spidxQuantity
.addIndex("IDX_SP_QUANTITY_HASH_SYSUN")
.unique(false)
.withColumns("HASH_IDENTITY_SYS_UNITS", "SP_VALUE");
spidxQuantity
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")))
.addCalculator("HASH_IDENTITY_AND_UNITS", t -> ResourceIndexedSearchParamQuantity.calculateHashUnits(t.getResourceType(), t.getString("SP_NAME"), t.getString("SP_UNITS")))
.addCalculator("HASH_IDENTITY_SYS_UNITS", t -> ResourceIndexedSearchParamQuantity.calculateHashSystemAndUnits(t.getResourceType(), t.getString("SP_NAME"), t.getString("SP_SYSTEM"), t.getString("SP_UNITS")))
);
// Indexes - String
Builder.BuilderWithTableName spidxString = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_STRING");
spidxString
.dropIndex("IDX_SP_STRING");
spidxString
.addColumn("HASH_NORM_PREFIX")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxString
.addColumn("HASH_NORM")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxString
.addIndex("IDX_SP_STRING_HASH_NRM")
.unique(false)
.withColumns("HASH_NORM_PREFIX", "SP_VALUE_NORMALIZED");
spidxString
.addIndex("IDX_SP_STRING_HASH_EXCT")
.unique(false)
.withColumns("HASH_EXACT");
spidxString
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("IDX_SP_STRING_HASH_NRM", t -> ResourceIndexedSearchParamString.calculateHashNormalized(new DaoConfig(), t.getResourceType(), t.getString("SP_NAME"), t.getString("SP_VALUE_NORMALIZED")))
.addCalculator("IDX_SP_STRING_HASH_EXCT", t -> ResourceIndexedSearchParamString.calculateHashExact(t.getResourceType(), t.getParamName(), t.getString("SP_VALUE_EXACT")))
);
// Indexes - Token
Builder.BuilderWithTableName spidxToken = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_TOKEN");
spidxToken
.dropIndex("IDX_SP_TOKEN");
spidxToken
.dropIndex("IDX_SP_TOKEN_UNQUAL");
spidxToken
.addColumn("HASH_IDENTITY")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxToken
.addColumn("HASH_SYS")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxToken
.addColumn("HASH_SYS_AND_VALUE")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxToken
.addColumn("HASH_VALUE")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxToken
.addIndex("IDX_SP_TOKEN_HASH")
.unique(false)
.withColumns("HASH_IDENTITY");
spidxToken
.addIndex("IDX_SP_TOKEN_HASH_S")
.unique(false)
.withColumns("HASH_SYS");
spidxToken
.addIndex("IDX_SP_TOKEN_HASH_SV")
.unique(false)
.withColumns("HASH_SYS_AND_VALUE");
spidxToken
.addIndex("IDX_SP_TOKEN_HASH_V")
.unique(false)
.withColumns("HASH_VALUE");
spidxToken
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")))
.addCalculator("HASH_SYS", t -> ResourceIndexedSearchParamToken.calculateHashSystem(t.getResourceType(), t.getParamName(), t.getString("SP_SYSTEM")))
.addCalculator("HASH_SYS_AND_VALUE", t -> ResourceIndexedSearchParamToken.calculateHashSystemAndValue(t.getResourceType(), t.getParamName(), t.getString("SP_SYSTEM"), t.getString("SP_VALUE")))
.addCalculator("HASH_VALUE", t -> ResourceIndexedSearchParamToken.calculateHashValue(t.getResourceType(), t.getParamName(), t.getString("SP_VALUE")))
);
// Indexes - URI
Builder.BuilderWithTableName spidxUri = forVersion(VersionEnum.V3_5_0).onTable("HFJ_SPIDX_URI");
spidxUri
.addColumn("HASH_IDENTITY")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spidxUri
.addIndex("IDX_SP_URI_HASH_IDENTITY")
.unique(false)
.withColumns("HASH_IDENTITY", "SP_URI");
spidxUri
.addIndex("IDX_SP_URI_HASH_URI")
.unique(false)
.withColumns("HASH_URI");
spidxUri
.addTask(new CalculateHashesTask()
.setColumnName("HASH_IDENTITY")
.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")))
.addCalculator("HASH_URI", t -> ResourceIndexedSearchParamUri.calculateHashUri(t.getResourceType(), t.getString("SP_NAME"), t.getString("SP_URI")))
);
// Search Parameter Presence
Builder.BuilderWithTableName spp = forVersion(VersionEnum.V3_5_0).onTable("HFJ_RES_PARAM_PRESENT");
spp.dropIndex("IDX_RESPARMPRESENT_SPID_RESID");
spp
.addColumn("HASH_PRESENCE")
.nullable()
.type(AddColumnTask.ColumnTypeEnum.LONG);
spp
.addIndex("IDX_RESPARMPRESENT_HASHPRES")
.unique(false)
.withColumns("HASH_PRESENCE");
ArbitrarySqlTask consolidateSearchParamPresenceIndexesTask = new ArbitrarySqlTask("Consolidate search parameter presence indexes");
consolidateSearchParamPresenceIndexesTask.setBatchSize(1);
String sql = "SELECT " +
"HFJ_SEARCH_PARM.RES_TYPE RES_TYPE, HFJ_SEARCH_PARM.PARAM_NAME PARAM_NAME, " +
"HFJ_RES_PARAM_PRESENT.PID PID, HFJ_RES_PARAM_PRESENT.SP_ID SP_ID, HFJ_RES_PARAM_PRESENT.SP_PRESENT SP_PRESENT, HFJ_RES_PARAM_PRESENT.HASH_PRESENT HASH_PRESENT " +
"from HFJ_RES_PARAM_PRESENT " +
"join HFJ_SEARCH_PARM ON (HFJ_SEARCH_PARM.PID = HFJ_RES_PARAM_PRESENT.SP_ID) " +
"where HFJ_RES_PARAM_PRESENT.HASH_PRESENT is null";
consolidateSearchParamPresenceIndexesTask.addQuery(sql, ArbitrarySqlTask.QueryModeEnum.BATCH_UNTIL_NO_MORE, t -> {
Long pid = (Long) t.get("PID");
Boolean present = (Boolean) t.get("SP_PRESENT");
String resType = (String) t.get("RES_TYPE");
String paramName = (String) t.get("PARAM_NAME");
Long hash = SearchParamPresent.calculateHashPresence(resType, paramName, present);
consolidateSearchParamPresenceIndexesTask.executeSql("update HFJ_RES_PARAM_PRESENT set HASH_PRESENCE = ? where PID = ?", hash, pid);
});
forVersion(VersionEnum.V3_5_0).addTask(consolidateSearchParamPresenceIndexesTask);
// Concept
Builder.BuilderWithTableName trmConcept = forVersion(VersionEnum.V3_5_0).onTable("TRM_CONCEPT");
trmConcept
.addIndex("IDX_CONCEPT_UPDATED")
.unique(false)
.withColumns("CONCEPT_UPDATED");
trmConcept
.modifyColumn("CODE")
.nonNullable()
.withType(BaseTableColumnTypeTask.ColumnTypeEnum.STRING, 500);
}
@ -74,6 +281,7 @@ public class HapiFhirJpaMigrationTasks {
private class BuilderWithTableName {
private String myIndexName;
private String myColumnName;
void dropIndex(String theIndexName) {
DropIndexTask task = new DropIndexTask();
@ -87,6 +295,21 @@ public class HapiFhirJpaMigrationTasks {
return new BuilderAddIndexWithName();
}
public BuilderAddColumnWithName addColumn(String theColumnName) {
myColumnName = theColumnName;
return new BuilderAddColumnWithName();
}
public void addTask(BaseTableTask<?> theTask) {
theTask.setTableName(myTableName);
Builder.this.addTask(theTask);
}
public BuilderModifyColumnWithName modifyColumn(String theColumnName) {
myColumnName = theColumnName;
return new BuilderModifyColumnWithName();
}
private class BuilderAddIndexWithName {
private boolean myUnique;
@ -106,6 +329,57 @@ public class HapiFhirJpaMigrationTasks {
}
}
}
private class BuilderAddColumnWithName {
private boolean myNullable;
public BuilderAddColumnWithNameNullable nullable() {
myNullable = true;
return new BuilderAddColumnWithNameNullable();
}
private class BuilderAddColumnWithNameNullable {
public void type(AddColumnTask.ColumnTypeEnum theColumnType) {
AddColumnTask task = new AddColumnTask();
task.setColumnName(myColumnName);
task.setNullable(myNullable);
task.setColumnType(theColumnType);
addTask(task);
}
}
}
private class BuilderModifyColumnWithName {
private boolean myNullable;
public BuilderModifyColumnWithNameAndNullable nullable() {
myNullable = true;
return new BuilderModifyColumnWithNameAndNullable();
}
public BuilderModifyColumnWithNameAndNullable nonNullable() {
myNullable = false;
return new BuilderModifyColumnWithNameAndNullable();
}
private class BuilderModifyColumnWithNameAndNullable {
public void withType(BaseTableColumnTypeTask.ColumnTypeEnum theColumnType, int theLength) {
if (theColumnType == BaseTableColumnTypeTask.ColumnTypeEnum.STRING) {
ModifyColumnTask task = new ModifyColumnTask();
task.setColumnName(myColumnName);
task.setTableName(myTableName);
task.setColumnLength(theLength);
task.setNullable(myNullable);
addTask(task);
} else {
throw new IllegalArgumentException("Can not specify length for column of type " + theColumnType);
}
}
}
}
}
}

44
hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTest.java Normal file
View File

@ -0,0 +1,44 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.migrate.JdbcUtils;
import org.junit.Test;
import java.sql.SQLException;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertThat;
public class AddColumnTest extends BaseTest {
@Test
public void testColumnDoesntAlreadyExist() throws SQLException {
executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255))");
AddColumnTask task = new AddColumnTask();
task.setTableName("SOMETABLE");
task.setColumnName("newcol");
task.setColumnType(AddColumnTask.ColumnTypeEnum.LONG);
task.setNullable(true);
getMigrator().addTask(task);
getMigrator().migrate();
assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "SOMETABLE"), containsInAnyOrder("PID", "TEXTCOL", "NEWCOL"));
}
@Test
public void testColumnAlreadyExists() throws SQLException {
executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255), newcol bigint)");
AddColumnTask task = new AddColumnTask();
task.setTableName("SOMETABLE");
task.setColumnName("newcol");
task.setColumnType(AddColumnTask.ColumnTypeEnum.LONG);
getMigrator().addTask(task);
getMigrator().migrate();
assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "SOMETABLE"), containsInAnyOrder("PID", "TEXTCOL", "NEWCOL"));
}
}

53
hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/ArbitrarySqlTest.java Normal file
View File

@ -0,0 +1,53 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.entity.SearchParamPresent;
import org.junit.Test;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
public class ArbitrarySqlTest extends BaseTest {
@Test
public void test350MigrateSearchParams() {
executeSql("create table HFJ_SEARCH_PARM (PID bigint not null, RES_TYPE varchar(255), PARAM_NAME varchar(255))");
executeSql("insert into HFJ_SEARCH_PARM (PID, RES_TYPE, PARAM_NAME) values (1, 'Patient', 'identifier')");
executeSql("insert into HFJ_SEARCH_PARM (PID, RES_TYPE, PARAM_NAME) values (2, 'Patient', 'family')");
executeSql("create table HFJ_RES_PARAM_PRESENT (PID bigint, SP_ID bigint, SP_PRESENT boolean, HASH_PRESENT bigint)");
executeSql("insert into HFJ_RES_PARAM_PRESENT (PID, SP_ID, SP_PRESENT, HASH_PRESENT) values (100, 1, true, null)");
executeSql("insert into HFJ_RES_PARAM_PRESENT (PID, SP_ID, SP_PRESENT, HASH_PRESENT) values (101, 2, true, null)");
ArbitrarySqlTask task = new ArbitrarySqlTask("Consolidate search parameter presence indexes");
task.setBatchSize(1);
String sql = "SELECT " +
"HFJ_SEARCH_PARM.RES_TYPE RES_TYPE, HFJ_SEARCH_PARM.PARAM_NAME PARAM_NAME, " +
"HFJ_RES_PARAM_PRESENT.PID PID, HFJ_RES_PARAM_PRESENT.SP_ID SP_ID, HFJ_RES_PARAM_PRESENT.SP_PRESENT SP_PRESENT, HFJ_RES_PARAM_PRESENT.HASH_PRESENT HASH_PRESENT " +
"from HFJ_RES_PARAM_PRESENT " +
"join HFJ_SEARCH_PARM ON (HFJ_SEARCH_PARM.PID = HFJ_RES_PARAM_PRESENT.SP_ID) " +
"where HFJ_RES_PARAM_PRESENT.HASH_PRESENT is null";
task.addQuery(sql, ArbitrarySqlTask.QueryModeEnum.BATCH_UNTIL_NO_MORE, t -> {
Long pid = (Long) t.get("PID");
Boolean present = (Boolean) t.get("SP_PRESENT");
String resType = (String) t.get("RES_TYPE");
String paramName = (String) t.get("PARAM_NAME");
Long hash = SearchParamPresent.calculateHashPresence(resType, paramName, present);
task.executeSql("update HFJ_RES_PARAM_PRESENT set HASH_PRESENT = ? where PID = ?", hash, pid);
});
getMigrator().addTask(task);
getMigrator().migrate();
List<Map<String, Object>> rows = executeQuery("select * from HFJ_RES_PARAM_PRESENT order by PID asc");
assertEquals(2, rows.size());
assertEquals(100L, rows.get(0).get("PID"));
assertEquals(-1100208805056022671L, rows.get(0).get("HASH_PRESENT"));
assertEquals(101L, rows.get(1).get("PID"));
assertEquals(-756348509333838170L, rows.get(1).get("HASH_PRESENT"));
}
}

14
hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/BaseTest.java
View File

@ -5,6 +5,10 @@ import ca.uhn.fhir.jpa.migrate.Migrator;
import org.intellij.lang.annotations.Language;
import org.junit.After;
import org.junit.Before;
import org.springframework.jdbc.core.ColumnMapRowMapper;
import java.util.List;
import java.util.Map;
public class BaseTest {
@ -22,13 +26,19 @@ public class BaseTest {
}
protected void executeSql(@Language("SQL") String theSql) {
protected void executeSql(@Language("SQL") String theSql, Object... theArgs) {
myConnectionProperties.getTxTemplate().execute(t -> {
myConnectionProperties.newJdbcTemplate().execute(theSql);
myConnectionProperties.newJdbcTemplate().update(theSql, theArgs);
return null;
});
}
protected List<Map<String, Object>> executeQuery(@Language("SQL") String theSql, Object... theArgs) {
return myConnectionProperties.getTxTemplate().execute(t -> {
return myConnectionProperties.newJdbcTemplate().query(theSql, theArgs, new ColumnMapRowMapper());
});
}
public Migrator getMigrator() {
return myMigrator;
}

53
hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/CreateHashesTest.java Normal file
View File

@ -0,0 +1,53 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
import org.junit.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import java.util.Map;
import static org.junit.Assert.assertEquals;
public class CreateHashesTest extends BaseTest {
@Test
public void testCreateHashes() {
executeSql("create table HFJ_SPIDX_TOKEN (SP_ID bigint not null, SP_MISSING boolean, SP_NAME varchar(100) not null, RES_ID bigint, RES_TYPE varchar(255) not null, SP_UPDATED timestamp, HASH_IDENTITY bigint, HASH_SYS bigint, HASH_SYS_AND_VALUE bigint, HASH_VALUE bigint, SP_SYSTEM varchar(200), SP_VALUE varchar(200), primary key (SP_ID))");
executeSql("insert into HFJ_SPIDX_TOKEN (SP_MISSING, SP_NAME, RES_ID, RES_TYPE, SP_UPDATED, SP_SYSTEM, SP_VALUE, SP_ID) values (false, 'identifier', 999, 'Patient', '2018-09-03 07:44:49.196', 'urn:oid:1.2.410.100110.10.41308301', '88888888', 1)");
executeSql("insert into HFJ_SPIDX_TOKEN (SP_MISSING, SP_NAME, RES_ID, RES_TYPE, SP_UPDATED, SP_SYSTEM, SP_VALUE, SP_ID) values (false, 'identifier', 999, 'Patient', '2018-09-03 07:44:49.196', 'urn:oid:1.2.410.100110.10.41308301', '99999999', 2)");
CalculateHashesTask task = new CalculateHashesTask();
task.setTableName("HFJ_SPIDX_TOKEN");
task.setColumnName("HASH_IDENTITY");
task.addCalculator("HASH_IDENTITY", t -> BaseResourceIndexedSearchParam.calculateHashIdentity(t.getResourceType(), t.getString("SP_NAME")));
task.addCalculator("HASH_SYS", t -> ResourceIndexedSearchParamToken.calculateHashSystem(t.getResourceType(), t.getParamName(), t.getString("SP_SYSTEM")));
task.addCalculator("HASH_SYS_AND_VALUE", t -> ResourceIndexedSearchParamToken.calculateHashSystemAndValue(t.getResourceType(), t.getParamName(), t.getString("SP_SYSTEM"), t.getString("SP_VALUE")));
task.addCalculator("HASH_VALUE", t -> ResourceIndexedSearchParamToken.calculateHashValue(t.getResourceType(), t.getParamName(), t.getString("SP_VALUE")));
task.setBatchSize(1);
getMigrator().addTask(task);
getMigrator().migrate();
getConnectionProperties().getTxTemplate().execute(t -> {
Map<String, Object> map;
JdbcTemplate jdbcTemplate = getConnectionProperties().newJdbcTemplate();
map = jdbcTemplate.queryForMap("select * from HFJ_SPIDX_TOKEN where SP_ID = 1");
assertEquals(7001889285610424179L, map.get("HASH_IDENTITY"));
assertEquals(2686400398917843456L, map.get("HASH_SYS"));
assertEquals(-3943098850992523411L, map.get("HASH_SYS_AND_VALUE"));
assertEquals(845040519142030272L, map.get("HASH_VALUE"));
map = jdbcTemplate.queryForMap("select * from HFJ_SPIDX_TOKEN where SP_ID = 2");
assertEquals(7001889285610424179L, map.get("HASH_IDENTITY"));
assertEquals(2686400398917843456L, map.get("HASH_SYS"));
assertEquals(-6583685191951870327L, map.get("HASH_SYS_AND_VALUE"));
assertEquals(8271382783311609619L, map.get("HASH_VALUE"));
return null;
});
}
}

32
hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/ModifyColumnTest.java Normal file
View File

@ -0,0 +1,32 @@
package ca.uhn.fhir.jpa.migrate.taskdef;
import ca.uhn.fhir.jpa.migrate.JdbcUtils;
import org.junit.Test;
import java.sql.SQLException;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
public class ModifyColumnTest extends BaseTest {
@Test
public void testColumnAlreadyExists() throws SQLException {
executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255), newcol bigint)");
ModifyColumnTask task = new ModifyColumnTask();
task.setTableName("SOMETABLE");
task.setColumnName("TEXTCOL");
task.setColumnType(AddColumnTask.ColumnTypeEnum.STRING);
task.setNullable(true);
task.setColumnLength(300);
getMigrator().addTask(task);
getMigrator().migrate();
assertEquals("varchar(300)", JdbcUtils.getColumnType(getConnectionProperties(), "SOMETABLE", "TEXTCOL"));
}
}

1
hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasksTest.java
View File

@ -9,5 +9,4 @@ public class HapiFhirJpaMigrationTasksTest {
new HapiFhirJpaMigrationTasks();
}
}

17
hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/ctx/DefaultProfileValidationSupport.java
View File

@ -1,8 +1,8 @@
package org.hl7.fhir.r4.hapi.ctx;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.rest.api.Constants;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -15,6 +15,7 @@ import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent;
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
@ -169,7 +170,7 @@ public class DefaultProfileValidationSupport implements IValidationSupport {
InputStreamReader reader = null;
if (inputStream != null) {
try {
reader = new InputStreamReader(inputStream, Charsets.UTF_8);
reader = new InputStreamReader(inputStream, Constants.CHARSET_UTF8);
Bundle bundle = theContext.newXmlParser().parseResource(Bundle.class, reader);
for (BundleEntryComponent next : bundle.getEntry()) {
@ -190,8 +191,14 @@ public class DefaultProfileValidationSupport implements IValidationSupport {
}
}
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(inputStream);
try {
if (reader != null) {
reader.close();
}
inputStream.close();
} catch (IOException e) {
ourLog.warn("Failure closing stream", e);
}
}
} else {
ourLog.warn("Unable to load resource: {}", theClasspath);
@ -202,7 +209,7 @@ public class DefaultProfileValidationSupport implements IValidationSupport {
ourLog.info("Loading structure definitions from classpath: {}", theClasspath);
InputStream valuesetText = DefaultProfileValidationSupport.class.getResourceAsStream(theClasspath);
if (valuesetText != null) {
InputStreamReader reader = new InputStreamReader(valuesetText, Charsets.UTF_8);
InputStreamReader reader = new InputStreamReader(valuesetText, Constants.CHARSET_UTF8);
Bundle bundle = theContext.newXmlParser().parseResource(Bundle.class, reader);
for (BundleEntryComponent next : bundle.getEntry()) {