From 0bd8084b5e4250b996d36defa995b06fe54debfb Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 16 Jun 2020 16:05:10 -0400 Subject: [PATCH 01/22] Additional fixes to migration tasks for MySQL. --- .../ca/uhn/fhir/jpa/migrate/JdbcUtils.java | 41 ++++++++++++++ .../jpa/migrate/taskdef/AddColumnTask.java | 5 +- .../migrate/taskdef/AddForeignKeyTask.java | 3 +- .../jpa/migrate/taskdef/DropColumnTask.java | 16 ++++++ .../migrate/taskdef/DropForeignKeyTask.java | 3 +- .../jpa/migrate/taskdef/DropIndexTask.java | 9 ++-- .../jpa/migrate/taskdef/ModifyColumnTask.java | 3 +- .../jpa/migrate/taskdef/RenameColumnTask.java | 19 ++++++- .../jpa/migrate/taskdef/RenameIndexTask.java | 3 ++ .../tasks/HapiFhirJpaMigrationTasks.java | 2 +- .../jpa/migrate/taskdef/DropColumnTest.java | 54 +++++++++++++++++++ 11 files changed, 147 insertions(+), 11 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java index 6c60358fa9a..53c80507168 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/JdbcUtils.java @@ -317,6 +317,47 @@ public class JdbcUtils { } } + /** + * Retrieve names of foreign keys that reference a specified foreign key column. + */ + public static Set getForeignKeysForColumn(DriverTypeEnum.ConnectionProperties theConnectionProperties, String theForeignKeyColumn, String theForeignTable) throws SQLException { + DataSource dataSource = Objects.requireNonNull(theConnectionProperties.getDataSource()); + + try (Connection connection = dataSource.getConnection()) { + return theConnectionProperties.getTxTemplate().execute(t -> { + DatabaseMetaData metadata; + try { + metadata = connection.getMetaData(); + String catalog = connection.getCatalog(); + String schema = connection.getSchema(); + + + List parentTables = new ArrayList<>(); + parentTables.addAll(JdbcUtils.getTableNames(theConnectionProperties)); + + String foreignTable = massageIdentifier(metadata, theForeignTable); + + Set fkNames = new HashSet<>(); + for (String nextParentTable : parentTables) { + ResultSet indexes = metadata.getCrossReference(catalog, schema, nextParentTable, catalog, schema, foreignTable); + + while (indexes.next()) { + if (theForeignKeyColumn.equals(indexes.getString("FKCOLUMN_NAME"))) { + String fkName = indexes.getString("FK_NAME"); + fkName = toUpperCase(fkName, Locale.US); + fkNames.add(fkName); + } + } + } + + return fkNames; + } catch (SQLException e) { + throw new InternalErrorException(e); + } + }); + } + } + /** * Retrieve all index names */ diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java index 4797f1cfefc..bf76353ce9a 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java @@ -53,9 +53,12 @@ public class AddColumnTask extends BaseTableColumnTypeTask { String sql; switch (getDriverType()) { + case MYSQL_5_7: + // Quote the column name as "SYSTEM" is a reserved word in MySQL + sql = "alter table " + getTableName() + " add column `" + getColumnName() + "` " + typeStatement; + break; case DERBY_EMBEDDED: case MARIADB_10_1: - case MYSQL_5_7: case POSTGRES_9_4: sql = "alter table " + getTableName() + " add column " + getColumnName() + " " + typeStatement; break; diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddForeignKeyTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddForeignKeyTask.java index 7cbe9832a34..61b4a28410f 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddForeignKeyTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddForeignKeyTask.java @@ -78,7 +78,8 @@ public class AddForeignKeyTask extends BaseTableColumnTask { switch (getDriverType()) { case MARIADB_10_1: case MYSQL_5_7: - sql = "alter table " + getTableName() + " add constraint " + myConstraintName + " foreign key (" + getColumnName() + ") references " + myForeignTableName + " (" + myForeignColumnName + ")"; + // Quote the column names as "SYSTEM" is a reserved word in MySQL + sql = "alter table " + getTableName() + " add constraint " + myConstraintName + " foreign key (`" + getColumnName() + "`) references " + myForeignTableName + " (`" + myForeignColumnName + "`)"; break; case POSTGRES_9_4: case DERBY_EMBEDDED: diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java index ff1acc30f15..8b7ff289526 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java @@ -20,12 +20,14 @@ package ca.uhn.fhir.jpa.migrate.taskdef; * #L% */ +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; import ca.uhn.fhir.jpa.migrate.JdbcUtils; import org.intellij.lang.annotations.Language; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; +import java.util.List; import java.util.Set; public class DropColumnTask extends BaseTableColumnTask { @@ -50,6 +52,20 @@ public class DropColumnTask extends BaseTableColumnTask { return; } + if(getDriverType().equals(DriverTypeEnum.MYSQL_5_7)) { + // Some DBs such as MYSQL require that foreign keys depending on the column be dropped before the column itself is dropped. + logInfo(ourLog, "Dropping any foreign keys on table {} depending on column {}", getTableName(), getColumnName()); + Set foreignKeys = JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), getColumnName(), getTableName()); + if(foreignKeys != null) { + for (String foreignKey:foreignKeys) { + List dropFkSqls = DropForeignKeyTask.generateSql(getTableName(), foreignKey, getDriverType()); + for(String dropFkSql : dropFkSqls) { + executeSql(getTableName(), dropFkSql); + } + } + } + } + String tableName = getTableName(); String columnName = getColumnName(); String sql = createSql(tableName, columnName); diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java index 33a78d4c29a..088682ffc1d 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java @@ -102,8 +102,7 @@ public class DropForeignKeyTask extends BaseTableTask { switch (theDriverType) { case MYSQL_5_7: // Lousy MYQL.... - sqls.add("alter table " + theTableName + " drop constraint " + theConstraintName); - sqls.add("alter table " + theTableName + " drop index " + theConstraintName); + sqls.add("alter table " + theTableName + " drop foreign key " + theConstraintName); break; case MARIADB_10_1: case POSTGRES_9_4: diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java index 3d6d20823d1..cbac30507c0 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java @@ -107,6 +107,9 @@ public class DropIndexTask extends BaseTableTask { // Drop constraint switch (theDriverType) { case MYSQL_5_7: + // Need to quote the index name as the word "PRIMARY" is reserved in MySQL + sql.add("alter table " + theTableName + " drop index `" + theIndexName + "`"); + break; case MARIADB_10_1: sql.add("alter table " + theTableName + " drop index " + theIndexName); break; @@ -114,16 +117,14 @@ public class DropIndexTask extends BaseTableTask { sql.add("drop index " + theIndexName); break; case DERBY_EMBEDDED: + case ORACLE_12C: + case MSSQL_2012: sql.add("alter table " + theTableName + " drop constraint " + theIndexName); break; case POSTGRES_9_4: sql.add("alter table " + theTableName + " drop constraint if exists " + theIndexName + " cascade"); sql.add("drop index if exists " + theIndexName + " cascade"); break; - case ORACLE_12C: - case MSSQL_2012: - sql.add("alter table " + theTableName + " drop constraint " + theIndexName); - break; } } else { // Drop index diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ModifyColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ModifyColumnTask.java index e21d008bccf..2ec4c08e5ef 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ModifyColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/ModifyColumnTask.java @@ -99,7 +99,8 @@ public class ModifyColumnTask extends BaseTableColumnTypeTask { break; case MARIADB_10_1: case MYSQL_5_7: - sql = "alter table " + getTableName() + " modify column " + getColumnName() + " " + type + notNull; + // Quote the column name as "SYSTEM" is a reserved word in MySQL + sql = "alter table " + getTableName() + " modify column `" + getColumnName() + "` " + type + notNull; break; case POSTGRES_9_4: if (!alreadyOfCorrectType) { diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java index 83e35200d97..57aee17fe56 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.migrate.taskdef; * #L% */ +import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; import ca.uhn.fhir.jpa.migrate.JdbcUtils; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import org.apache.commons.lang3.Validate; @@ -30,6 +31,7 @@ import org.springframework.jdbc.core.ColumnMapRowMapper; import org.springframework.jdbc.core.JdbcTemplate; import java.sql.SQLException; +import java.util.List; import java.util.Set; public class RenameColumnTask extends BaseTableTask { @@ -82,6 +84,20 @@ public class RenameColumnTask extends BaseTableTask { throw new SQLException("Can not rename " + getTableName() + "." + myOldName + " to " + myNewName + " because both columns exist and data exists in " + myNewName); } + if (getDriverType().equals(DriverTypeEnum.MYSQL_5_7)) { + // Some DBs such as MYSQL require that foreign keys depending on the column be dropped before the column itself is dropped. + logInfo(ourLog, "Table {} has columns {} and {} - Going to drop any foreign keys depending on column {} before renaming", getTableName(), myOldName, myNewName, myNewName); + Set foreignKeys = JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), myNewName, getTableName()); + if(foreignKeys != null) { + for (String foreignKey:foreignKeys) { + List dropFkSqls = DropForeignKeyTask.generateSql(getTableName(), foreignKey, getDriverType()); + for(String dropFkSql : dropFkSqls) { + executeSql(getTableName(), dropFkSql); + } + } + } + } + logInfo(ourLog, "Table {} has columns {} and {} - Going to drop {} before renaming", getTableName(), myOldName, myNewName, myNewName); String sql = DropColumnTask.createSql(getTableName(), myNewName); executeSql(getTableName(), sql); @@ -124,7 +140,8 @@ public class RenameColumnTask extends BaseTableTask { sql = "ALTER TABLE " + getTableName() + " CHANGE COLUMN " + myOldName + " TO " + myNewName; break; case MYSQL_5_7: - sql = "ALTER TABLE " + getTableName() + " CHANGE COLUMN " + myOldName + " " + myNewName + " " + theExistingType + " " + theExistingNotNull; + // Quote the column names as "SYSTEM" is a reserved word in MySQL + sql = "ALTER TABLE " + getTableName() + " CHANGE COLUMN `" + myOldName + "` `" + myNewName + "` " + theExistingType + " " + theExistingNotNull; break; case POSTGRES_9_4: case ORACLE_12C: diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java index 630733ef3f2..b2edd22caf2 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java @@ -110,6 +110,9 @@ public class RenameIndexTask extends BaseTableTask { // Drop constraint switch (theDriverType) { case MYSQL_5_7: + // Quote the index names as "PRIMARY" is a reserved word in MySQL + sql.add("rename index `" + theOldIndexName + "` to `" + theNewIndexName + "`"); + break; case MARIADB_10_1: case DERBY_EMBEDDED: sql.add("rename index " + theOldIndexName + " to " + theNewIndexName); diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index 9fd3f7ab866..83a6b9747fb 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -118,7 +118,7 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { pkgVerRes.addColumn("RES_SIZE_BYTES").nonNullable().type(ColumnTypeEnum.LONG); pkgVerRes.addColumn("UPDATED_TIME").nonNullable().type(ColumnTypeEnum.DATE_TIMESTAMP); pkgVerRes.addForeignKey("20200610.11", "FK_NPM_PACKVERRES_PACKVER").toColumn("PACKVER_PID").references("NPM_PACKAGE_VER", "PID"); - pkgVerRes.addForeignKey("20200610.12", "FK_NPM_PKVR_RESID").toColumn("BINARY_RES_ID").references("HFJ_RESOURCE", "PID"); + pkgVerRes.addForeignKey("20200610.12", "FK_NPM_PKVR_RESID").toColumn("BINARY_RES_ID").references("HFJ_RESOURCE", "RES_ID"); pkgVerRes.addIndex("20200610.13", "IDX_PACKVERRES_URL").unique(false).withColumns("CANONICAL_URL"); } diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java index e5b399df3e0..af63356aef5 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java @@ -7,6 +7,8 @@ import java.sql.SQLException; import java.util.function.Supplier; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertThat; public class DropColumnTest extends BaseTest { @@ -34,4 +36,56 @@ public class DropColumnTest extends BaseTest { } + @Test + public void testDropForeignKeyColumn() throws SQLException { + executeSql("create table PARENT (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); + executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PID)"); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + + DropColumnTask task = new DropColumnTask("1", "1"); + task.setTableName("CHILD"); + task.setColumnName("PARENTREF"); + getMigrator().addTask(task); + + getMigrator().migrate(); + + assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "CHILD"), containsInAnyOrder("PID")); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), empty()); + + // Do it again to make sure there is no error + getMigrator().migrate(); + getMigrator().migrate(); + + } + + + /* + executeSql("create table PARENT (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); + executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PID)"); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + + DropForeignKeyTask task = new DropForeignKeyTask("1", "1"); + task.setTableName("CHILD"); + task.setParentTableName("PARENT"); + task.setConstraintName("FK_MOM"); + getMigrator().addTask(task); + + getMigrator().migrate(); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), empty()); + + // Make sure additional calls don't crash + getMigrator().migrate(); + getMigrator().migrate(); + + */ } From 2d857f484c4a3bfbc04c7ed3d4a792d72b8705c9 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 16 Jun 2020 17:13:00 -0400 Subject: [PATCH 02/22] Fix broken JUnit. --- .../jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java index 738d231afb1..9c4fec4fe74 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java @@ -8,7 +8,7 @@ public class RenameColumnTaskDbSpecificTest { @Test public void testBuildSqlStatementForMySql() { - assertEquals("ALTER TABLE SOMETABLE CHANGE COLUMN myTextCol TEXTCOL integer null", createRenameColumnSql(DriverTypeEnum.MYSQL_5_7)); + assertEquals("ALTER TABLE SOMETABLE CHANGE COLUMN `myTextCol` `TEXTCOL` integer null", createRenameColumnSql(DriverTypeEnum.MYSQL_5_7)); } private String createRenameColumnSql(DriverTypeEnum theDriverTypeEnum) { From cc7ed1b036061b8c04e58b57bbe73ae2ea5d2a67 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Tue, 16 Jun 2020 17:21:17 -0400 Subject: [PATCH 03/22] Cleanup JUnit test. --- .../jpa/migrate/taskdef/DropColumnTest.java | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java index af63356aef5..24de53ed144 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java @@ -63,29 +63,4 @@ public class DropColumnTest extends BaseTest { } - - /* - executeSql("create table PARENT (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); - executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); - executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PID)"); - - assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); - - assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); - - DropForeignKeyTask task = new DropForeignKeyTask("1", "1"); - task.setTableName("CHILD"); - task.setParentTableName("PARENT"); - task.setConstraintName("FK_MOM"); - getMigrator().addTask(task); - - getMigrator().migrate(); - - assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), empty()); - - // Make sure additional calls don't crash - getMigrator().migrate(); - getMigrator().migrate(); - - */ } From eb990e9a68a855da05a7cfb1d268f12ad208f92d Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Wed, 17 Jun 2020 09:05:53 -0400 Subject: [PATCH 04/22] Enhance added JUnit test. --- .../uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java index 24de53ed144..b144aed0a2c 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTest.java @@ -39,12 +39,16 @@ public class DropColumnTest extends BaseTest { @Test public void testDropForeignKeyColumn() throws SQLException { executeSql("create table PARENT (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); - executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); + executeSql("create table SIBLING (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint, SIBLINGREF bigint)"); executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PID)"); + executeSql("alter table CHILD add constraint FK_BROTHER foreign key (SIBLINGREF) references SIBLING(PID)"); assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "SIBLING", "CHILD"), hasSize(1)); assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "SIBLINGREF", "CHILD"), containsInAnyOrder("FK_BROTHER")); DropColumnTask task = new DropColumnTask("1", "1"); task.setTableName("CHILD"); @@ -53,9 +57,13 @@ public class DropColumnTest extends BaseTest { getMigrator().migrate(); - assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "CHILD"), containsInAnyOrder("PID")); + assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "CHILD"), containsInAnyOrder("PID", "SIBLINGREF")); assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), empty()); + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "SIBLING", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), empty()); + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "SIBLINGREF", "CHILD"), containsInAnyOrder("FK_BROTHER")); // Do it again to make sure there is no error getMigrator().migrate(); From 5d22af45bf60ea904b6071ca75d2ac138ab4a0c6 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Wed, 17 Jun 2020 11:56:20 -0400 Subject: [PATCH 05/22] Added tests for cases where a renamed column is used in a Foreign Key constraint. --- .../jpa/migrate/taskdef/RenameColumnTask.java | 12 +- .../migrate/taskdef/RenameColumnTaskTest.java | 116 ++++++++++++++++++ 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java index 57aee17fe56..7e40827ffa1 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java @@ -23,6 +23,7 @@ package ca.uhn.fhir.jpa.migrate.taskdef; import ca.uhn.fhir.jpa.migrate.DriverTypeEnum; import ca.uhn.fhir.jpa.migrate.JdbcUtils; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.slf4j.Logger; @@ -42,6 +43,8 @@ public class RenameColumnTask extends BaseTableTask { private boolean myIsOkayIfNeitherColumnExists; private boolean myDeleteTargetColumnFirstIfBothExist; + private boolean mySimulateMySQLForTest = false; + public RenameColumnTask(String theProductVersion, String theSchemaVersion) { super(theProductVersion, theSchemaVersion); } @@ -84,8 +87,8 @@ public class RenameColumnTask extends BaseTableTask { throw new SQLException("Can not rename " + getTableName() + "." + myOldName + " to " + myNewName + " because both columns exist and data exists in " + myNewName); } - if (getDriverType().equals(DriverTypeEnum.MYSQL_5_7)) { - // Some DBs such as MYSQL require that foreign keys depending on the column be dropped before the column itself is dropped. + if (getDriverType().equals(DriverTypeEnum.MYSQL_5_7) || mySimulateMySQLForTest) { + // Some DBs such as MYSQL require that foreign keys depending on the column be explicitly dropped before the column itself is dropped. logInfo(ourLog, "Table {} has columns {} and {} - Going to drop any foreign keys depending on column {} before renaming", getTableName(), myOldName, myNewName, myNewName); Set foreignKeys = JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), myNewName, getTableName()); if(foreignKeys != null) { @@ -173,4 +176,9 @@ public class RenameColumnTask extends BaseTableTask { theBuilder.append(myOldName); theBuilder.append(myNewName); } + + @VisibleForTesting + void setSimulateMySQLForTest(boolean theSimulateMySQLForTest) { + mySimulateMySQLForTest = theSimulateMySQLForTest; + } } diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java index f1927501f69..f4f2dddf4f4 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java @@ -8,7 +8,10 @@ import java.sql.SQLException; import java.util.Set; import java.util.function.Supplier; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -35,6 +38,41 @@ public class RenameColumnTaskTest extends BaseTest { assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "SOMETABLE"), containsInAnyOrder("PID", "TEXTCOL")); } + @Test + public void testForeignKeyColumnAlreadyExists_MySql() throws SQLException { + testForeignKeyColumnAlreadyExists(true); + } + + private void testForeignKeyColumnAlreadyExists(boolean isMySql) throws SQLException { + executeSql("create table PARENT (PID bigint not null, TEXTCOL varchar(255), primary key (PID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); + executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PID)"); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + + RenameColumnTask task = new RenameColumnTask("1", "1"); + task.setTableName("CHILD"); + task.setOldName("myParentRef"); + task.setNewName("PARENTREF"); + task.setSimulateMySQLForTest(isMySql); + getMigrator().addTask(task); + + getMigrator().migrate(); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "CHILD"), containsInAnyOrder("PID", "PARENTREF")); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + } + + @Test + public void testForeignKeyColumnAlreadyExists_OtherDB() throws SQLException { + testForeignKeyColumnAlreadyExists(false); + } + @Test public void testBothExistDeleteTargetFirst() throws SQLException { executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255), myTextCol varchar(255))"); @@ -53,6 +91,48 @@ public class RenameColumnTaskTest extends BaseTest { assertThat(columnNames.toString(), columnNames, containsInAnyOrder("PID", "TEXTCOL")); } + @Test + public void testForeignKeyColumnBothExistDeleteTargetFirst_MySql() throws SQLException { + testForeignKeyColumnBothExistDeleteTargetFirst(true); + } + + private void testForeignKeyColumnBothExistDeleteTargetFirst(boolean isMySql) throws SQLException { + executeSql("create table PARENT (PARENTID bigint not null, TEXTCOL varchar(255), primary key (PARENTID))"); + executeSql("create table RELATION (RELATIONID bigint not null, TEXTCOL varchar(255), primary key (RELATIONID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint, NOKREF bigint)"); + executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PARENTID)"); + executeSql("alter table CHILD add constraint FK_NOK foreign key (NOKREF) references RELATION(RELATIONID)"); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "RELATION", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "NOKREF", "CHILD"), containsInAnyOrder("FK_NOK")); + + RenameColumnTask task = new RenameColumnTask("1", "1"); + task.setTableName("CHILD"); + task.setOldName("PARENTREF"); + task.setNewName("NOKREF"); + task.setDeleteTargetColumnFirstIfBothExist(true); + task.setSimulateMySQLForTest(isMySql); + getMigrator().addTask(task); + + getMigrator().migrate(); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "RELATION", "CHILD"), empty()); + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "CHILD"), containsInAnyOrder("PID", "NOKREF")); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "NOKREF", "CHILD"), containsInAnyOrder("FK_MOM")); + + } + + @Test + public void testForeignKeyColumnBothExistDeleteTargetFirst_OtherDB() throws SQLException { + testForeignKeyColumnBothExistDeleteTargetFirst(false); + } + @Test public void testBothExistDeleteTargetFirstDataExistsInSourceAndTarget() throws SQLException { executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255), myTextCol varchar(255))"); @@ -91,6 +171,42 @@ public class RenameColumnTaskTest extends BaseTest { assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "SOMETABLE"), containsInAnyOrder("PID", "TEXTCOL")); } + @Test + public void testForeignKeyColumnDoesntAlreadyExist_MySql() throws SQLException { + testForeignKeyColumnDoesntAlreadyExist(true); + } + + private void testForeignKeyColumnDoesntAlreadyExist(boolean isMySql) throws SQLException { + executeSql("create table PARENT (PARENTID bigint not null, TEXTCOL varchar(255), primary key (PARENTID))"); + executeSql("create table CHILD (PID bigint not null, PARENTREF bigint)"); + executeSql("alter table CHILD add constraint FK_MOM foreign key (PARENTREF) references PARENT(PARENTID)"); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "PARENTREF", "CHILD"), containsInAnyOrder("FK_MOM")); + + RenameColumnTask task = new RenameColumnTask("1", "1"); + task.setTableName("CHILD"); + task.setOldName("PARENTREF"); + task.setNewName("MOMREF"); + task.setSimulateMySQLForTest(isMySql); + getMigrator().addTask(task); + + getMigrator().migrate(); + + assertThat(JdbcUtils.getForeignKeys(getConnectionProperties(), "PARENT", "CHILD"), hasSize(1)); + + assertThat(JdbcUtils.getColumnNames(getConnectionProperties(), "CHILD"), containsInAnyOrder("PID", "MOMREF")); + + assertThat(JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), "MOMREF", "CHILD"), containsInAnyOrder("FK_MOM")); + + } + + @Test + public void testForeignKeyColumnDoesntAlreadyExist_OtherDB() throws SQLException { + testForeignKeyColumnDoesntAlreadyExist(false); + } + @Test public void testNeitherColumnExists() { executeSql("create table SOMETABLE (PID bigint not null)"); From 5b120ee71e752d97098b2b6377199c502f79d948 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Wed, 17 Jun 2020 11:57:37 -0400 Subject: [PATCH 06/22] Removed unused code and import statement. --- .../ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java index f4f2dddf4f4..61d06e27d66 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskTest.java @@ -8,7 +8,6 @@ import java.sql.SQLException; import java.util.Set; import java.util.function.Supplier; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; @@ -134,7 +133,7 @@ public class RenameColumnTaskTest extends BaseTest { } @Test - public void testBothExistDeleteTargetFirstDataExistsInSourceAndTarget() throws SQLException { + public void testBothExistDeleteTargetFirstDataExistsInSourceAndTarget() { executeSql("create table SOMETABLE (PID bigint not null, TEXTCOL varchar(255), myTextCol varchar(255))"); executeSql("INSERT INTO SOMETABLE (PID, TEXTCOL, myTextCol) VALUES (123, 'AAA', 'BBB')"); From b6540064ea84e234ab53d1588abd4ec4ff1a184c Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 17 Jun 2020 14:04:12 -0400 Subject: [PATCH 07/22] Improve TX scoping for validation (#1925) * Improve TX scoping for validation * Test fix * Test fix * Some cleanup * Test fixes * Test fix --- hapi-fhir-elasticsearch-6/pom.xml | 4 +- hapi-fhir-jpaserver-base/pom.xml | 6 + .../fhir/jpa/dao/BaseHapiFhirResourceDao.java | 1 + .../ca/uhn/fhir/jpa/entity/TermConcept.java | 4 - .../fhir/jpa/packages/JpaPackageCache.java | 2 +- .../fhir/jpa/term/BaseTermReadSvcImpl.java | 18 ++- .../term/TermCodeSystemStorageSvcImpl.java | 21 +-- .../uhn/fhir/jpa/term/TermReadSvcDstu3.java | 2 +- .../ca/uhn/fhir/jpa/term/TermReadSvcR4.java | 2 +- .../ca/uhn/fhir/jpa/term/TermReadSvcR5.java | 2 +- .../uhn/fhir/jpa/config/TestDstu2Config.java | 5 + .../uhn/fhir/jpa/config/TestDstu3Config.java | 5 + .../ca/uhn/fhir/jpa/config/TestR4Config.java | 4 + .../ca/uhn/fhir/jpa/config/TestR5Config.java | 5 + .../java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java | 8 +- .../r4/FhirResourceDaoR4QueryCountTest.java | 2 +- .../dao/r4/FhirResourceDaoR4ValidateTest.java | 148 +++++++++++++++++- ...stDstu3.java => IgInstallerDstu3Test.java} | 11 +- .../jpa/packages/JpaPackageCacheTest.java | 2 +- .../uhn/fhir/jpa/util/JpaClasspathTest.java | 25 +++ .../support/ValidationSupportChain.java | 8 + .../validator/FhirInstanceValidator.java | 4 +- 22 files changed, 253 insertions(+), 36 deletions(-) rename hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/{IgInstallerTestDstu3.java => IgInstallerDstu3Test.java} (94%) create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/util/JpaClasspathTest.java diff --git a/hapi-fhir-elasticsearch-6/pom.xml b/hapi-fhir-elasticsearch-6/pom.xml index 495be468d38..e56854ab3bb 100644 --- a/hapi-fhir-elasticsearch-6/pom.xml +++ b/hapi-fhir-elasticsearch-6/pom.xml @@ -6,9 +6,9 @@ ca.uhn.hapi.fhir - hapi-fhir + hapi-deployable-pom 5.1.0-SNAPSHOT - ../pom.xml + ../hapi-deployable-pom/pom.xml hapi-fhir-elasticsearch-6 diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 74e37ced357..47e46699a3a 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -150,6 +150,12 @@ hapi-fhir-elasticsearch-6 ${project.version} shaded6 + + + org.apache.logging.log4j + log4j-api + + diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java index d19c9c4b229..79bffd0d49c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java @@ -1395,6 +1395,7 @@ public abstract class BaseHapiFhirResourceDao extends B } @Override + @Transactional(propagation = Propagation.SUPPORTS) public MethodOutcome validate(T theResource, IIdType theId, String theRawResource, EncodingEnum theEncoding, ValidationModeEnum theMode, String theProfile, RequestDetails theRequest) { if (theRequest != null) { ActionRequestDetails requestDetails = new ActionRequestDetails(theRequest, theResource, null, theId); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java index 7373c8e9952..798f847a633 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermConcept.java @@ -412,8 +412,4 @@ public class TermConcept implements Serializable { return getChildren().stream().map(t -> t.getChild()).collect(Collectors.toList()); } - - public VersionIndependentConcept toVersionIndependentConcept() { - return new VersionIndependentConcept(myCodeSystem.getCodeSystem().getCodeSystemUri(), myCode); - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java index 6b71e5f0258..e209831e7b3 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/packages/JpaPackageCache.java @@ -408,7 +408,7 @@ public class JpaPackageCache extends BasePackageCacheManager implements IHapiPac .build() .execute(new HttpGet(thePackageUrl))) { if (request.getStatusLine().getStatusCode() != 200) { - throw new IOException("Received HTTP " + request.getStatusLine().getStatusCode()); + throw new ResourceNotFoundException("Received HTTP " + request.getStatusLine().getStatusCode() + " from URL: " + thePackageUrl); } return IOUtils.toByteArray(request.getEntity().getContent()); } catch (IOException e) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java index 19544975083..3acbf6ba8eb 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/BaseTermReadSvcImpl.java @@ -89,6 +89,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; +import net.bytebuddy.implementation.bytecode.Throw; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.time.DateUtils; @@ -177,6 +178,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc { private static final TermCodeSystemVersion NO_CURRENT_VERSION = new TermCodeSystemVersion().setId(-1L); private static boolean ourLastResultsFromTranslationCache; // For testing. private static boolean ourLastResultsFromTranslationWithReverseCache; // For testing. + private static Runnable myInvokeOnNextCallForUnitTest; private final int myFetchSize = DEFAULT_FETCH_SIZE; private final Cache myCodeSystemCurrentVersionCache = Caffeine.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES).build(); @Autowired @@ -1299,7 +1301,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc { @Nullable private TermCodeSystemVersion getCurrentCodeSystemVersion(String theUri) { - TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theUri, uri -> { + TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theUri, uri -> myTxTemplate.execute(tx -> { TermCodeSystemVersion csv = null; TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(uri); if (cs != null && cs.getCurrentVersion() != null) { @@ -1310,7 +1312,7 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc { } else { return NO_CURRENT_VERSION; } - }); + })); if (retVal == NO_CURRENT_VERSION) { return null; } @@ -1935,8 +1937,15 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc { } @Override + @Transactional public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) { + if (myInvokeOnNextCallForUnitTest != null) { + Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest; + myInvokeOnNextCallForUnitTest = null; + invokeOnNextCallForUnitTest.run(); + } + IPrimitiveType urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class); String url = urlPrimitive.getValueAsString(); if (isNotBlank(url)) { @@ -2106,6 +2115,11 @@ public abstract class BaseTermReadSvcImpl implements ITermReadSvc { } } + @VisibleForTesting + public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) { + myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest; + } + static List toPersistedConcepts(List theConcept, TermCodeSystemVersion theCodeSystemVersion) { ArrayList retVal = new ArrayList<>(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java index 331c7aa2f24..0b971e13abf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermCodeSystemStorageSvcImpl.java @@ -447,16 +447,19 @@ public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc { doDelete(descriptor, loader, counter, myConceptDao); } - Optional codeSystemOpt = myCodeSystemDao.findWithCodeSystemVersionAsCurrentVersion(theCodeSystemVersionPid); - if (codeSystemOpt.isPresent()) { - TermCodeSystem codeSystem = codeSystemOpt.get(); - ourLog.info(" * Removing code system version {} as current version of code system {}", theCodeSystemVersionPid, codeSystem.getPid()); - codeSystem.setCurrentVersion(null); - myCodeSystemDao.save(codeSystem); - } + TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); + txTemplate.executeWithoutResult(tx -> { + Optional codeSystemOpt = myCodeSystemDao.findWithCodeSystemVersionAsCurrentVersion(theCodeSystemVersionPid); + if (codeSystemOpt.isPresent()) { + TermCodeSystem codeSystem = codeSystemOpt.get(); + ourLog.info(" * Removing code system version {} as current version of code system {}", theCodeSystemVersionPid, codeSystem.getPid()); + codeSystem.setCurrentVersion(null); + myCodeSystemDao.save(codeSystem); + } - ourLog.info(" * Deleting code system version"); - myCodeSystemVersionDao.deleteById(theCodeSystemVersionPid); + ourLog.info(" * Deleting code system version"); + myCodeSystemVersionDao.deleteById(theCodeSystemVersionPid); + }); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcDstu3.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcDstu3.java index 52280d3488a..dadcecf2dd1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcDstu3.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcDstu3.java @@ -144,7 +144,7 @@ public class TermReadSvcDstu3 extends BaseTermReadSvcImpl implements IValidation if (!haveValidated) { TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> c.toVersionIndependentConcept())); + codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> new VersionIndependentConcept(theCodeSystem, c.getCode()))); } if (codeOpt != null && codeOpt.isPresent()) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR4.java index 1eb9bac2a9c..479ea8e85a8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR4.java @@ -112,7 +112,7 @@ public class TermReadSvcR4 extends BaseTermReadSvcImpl implements ITermReadSvcR4 if (!haveValidated) { TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> c.toVersionIndependentConcept())); + codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> new VersionIndependentConcept(theCodeSystem, c.getCode()))); } if (codeOpt != null && codeOpt.isPresent()) { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR5.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR5.java index bbf943012e6..a43b825186c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR5.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermReadSvcR5.java @@ -99,7 +99,7 @@ public class TermReadSvcR5 extends BaseTermReadSvcImpl implements IValidationSup if (!haveValidated) { TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> c.toVersionIndependentConcept())); + codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> new VersionIndependentConcept(theCodeSystem, c.getCode()))); } if (codeOpt != null && codeOpt.isPresent()) { diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu2Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu2Config.java index ee52d38d72e..d73f87ef800 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu2Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu2Config.java @@ -41,6 +41,11 @@ public class TestDstu2Config extends BaseJavaConfigDstu2 { * starvation */ ourMaxThreads = (int) (Math.random() * 6.0) + 1; + + if ("true".equals(System.getProperty("single_db_connection"))) { + ourMaxThreads = 1; + } + } private Exception myLastStackTrace; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java index aa20363697d..73b3a49af62 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java @@ -94,6 +94,11 @@ public class TestDstu3Config extends BaseJavaConfigDstu3 { * starvation */ int maxThreads = (int) (Math.random() * 6.0) + 1; + + if ("true".equals(System.getProperty("single_db_connection"))) { + maxThreads = 1; + } + retVal.setMaxTotal(maxThreads); return retVal; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java index f479832e04f..9951d7cbc7a 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java @@ -43,6 +43,10 @@ public class TestR4Config extends BaseJavaConfigR4 { */ if (ourMaxThreads == null) { ourMaxThreads = (int) (Math.random() * 6.0) + 1; + + if ("true".equals(System.getProperty("single_db_connection"))) { + ourMaxThreads = 1; + } } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR5Config.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR5Config.java index 90a359ea404..41817800839 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR5Config.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR5Config.java @@ -41,6 +41,11 @@ public class TestR5Config extends BaseJavaConfigR5 { */ if (ourMaxThreads == null) { ourMaxThreads = (int) (Math.random() * 6.0) + 1; + + if ("true".equals(System.getProperty("single_db_connection"))) { + ourMaxThreads = 1; + } + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java index 66e08269cfc..0ab2b3728d9 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/BaseJpaTest.java @@ -43,6 +43,7 @@ import org.hibernate.HibernateException; import org.hibernate.Session; import org.hibernate.SessionFactory; import org.hibernate.jdbc.Work; +import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent; import org.hl7.fhir.dstu3.model.Resource; import org.hl7.fhir.instance.model.api.IBaseBundle; @@ -133,7 +134,8 @@ public abstract class BaseJpaTest extends BaseTest { @Qualifier(BaseConfig.JPA_VALIDATION_SUPPORT) @Autowired private IValidationSupport myJpaPersistedValidationSupport; - + @Autowired + private FhirInstanceValidator myFhirInstanceValidator; @After public void afterPerformCleanup() { @@ -150,7 +152,9 @@ public abstract class BaseJpaTest extends BaseTest { if (myJpaPersistedValidationSupport != null) { ProxyUtil.getSingletonTarget(myJpaPersistedValidationSupport, JpaPersistedResourceValidationSupport.class).clearCaches(); } - + if (myFhirInstanceValidator != null) { + myFhirInstanceValidator.invalidateCaches(); + } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java index e16923be4f3..d5569612515 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java @@ -145,7 +145,7 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { // Validate once myCaptureQueriesListener.clear(); myObservationDao.validate(obs, null, null, null, null, null, null); - assertEquals(myCaptureQueriesListener.logSelectQueriesForCurrentThread(), 10, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size()); + assertEquals(myCaptureQueriesListener.logSelectQueriesForCurrentThread(), 9, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size()); assertEquals(myCaptureQueriesListener.logUpdateQueriesForCurrentThread(), 0, myCaptureQueriesListener.getUpdateQueriesForCurrentThread().size()); assertEquals(myCaptureQueriesListener.logInsertQueriesForCurrentThread(), 0, myCaptureQueriesListener.getInsertQueriesForCurrentThread().size()); assertEquals(myCaptureQueriesListener.logDeleteQueriesForCurrentThread(), 0, myCaptureQueriesListener.getDeleteQueriesForCurrentThread().size()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java index f143c920ad4..cd38f6ab409 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java @@ -1,12 +1,16 @@ package ca.uhn.fhir.jpa.dao.r4; +import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.term.BaseTermReadSvcImpl; import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvc; import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet; +import ca.uhn.fhir.jpa.validation.JpaValidationSupportChain; import ca.uhn.fhir.rest.api.EncodingEnum; import ca.uhn.fhir.rest.api.MethodOutcome; import ca.uhn.fhir.rest.api.ValidationModeEnum; @@ -23,9 +27,27 @@ import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; import org.hl7.fhir.instance.model.api.IBaseOperationOutcome; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.AllergyIntolerance; +import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.CanonicalType; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Condition; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.Narrative; +import org.hl7.fhir.r4.model.Observation; import org.hl7.fhir.r4.model.Observation.ObservationStatus; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Organization; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Questionnaire; +import org.hl7.fhir.r4.model.QuestionnaireResponse; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.StructureDefinition; +import org.hl7.fhir.r4.model.ValueSet; import org.hl7.fhir.r5.utils.IResourceValidator; import org.junit.After; import org.junit.AfterClass; @@ -33,6 +55,7 @@ import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.AopTestUtils; +import org.springframework.transaction.PlatformTransactionManager; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -45,6 +68,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4ValidateTest.class); @@ -56,6 +82,10 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { private ITermCodeSystemStorageSvc myTermCodeSystemStorageSvcc; @Autowired private DaoRegistry myDaoRegistry; + @Autowired + private JpaValidationSupportChain myJpaValidationSupportChain; + @Autowired + private PlatformTransactionManager myTransactionManager; /** * Create a loinc valueset that expands to more results than the expander is willing to do @@ -155,13 +185,11 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { myCaptureQueriesListener.logSelectQueriesForCurrentThread(); - - } /** * Per: https://chat.fhir.org/#narrow/stream/179166-implementers/topic/Handling.20incomplete.20CodeSystems - * + *

* We should generate a warning if a code can't be found but the codesystem is a fragment */ @Test @@ -219,7 +247,6 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { } } - /** * Create a loinc valueset that expands to more results than the expander is willing to do * in memory, and make sure we can still validate correctly, even if we're using @@ -303,6 +330,115 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { } + /** + * Make sure that we do something sane when validating throws an unexpected exception + */ + @Test + public void testValidate_ValidationSupportThrowsException() { + IValidationSupport validationSupport = mock(IValidationSupport.class); + when(validationSupport.validateCodeInValueSet(any(), any(), any(), any(), any(), any())).thenAnswer(t -> { + // This will fail with a constraint error + try { + myResourceTableDao.save(new ResourceTable()); + myResourceTableDao.flush(); + } catch (Exception e) { + ourLog.info("Hit expected exception: {}", e.toString()); + } + return null; + }); + when(validationSupport.getFhirContext()).thenReturn(myFhirCtx); + + myJpaValidationSupportChain.addValidationSupport(0, validationSupport); + try { + + Observation obs = new Observation(); + obs.getText().setDivAsString("

Hello
"); + obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs"); + obs.setSubject(new Reference("Patient/123")); + obs.addPerformer(new Reference("Practitioner/123")); + obs.setEffective(DateTimeType.now()); + obs.setStatus(ObservationStatus.FINAL); + obs.setValue(new StringType("This is the value")); + + OperationOutcome oo; + + // Valid code + obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED); + obs.getCode().getCodingFirstRep().setSystem("http://loinc.org").setCode("CODE3").setDisplay("Display 3"); + oo = validateAndReturnOutcome(obs); + assertEquals(encode(oo), "No issues detected during validation", oo.getIssueFirstRep().getDiagnostics()); + + } finally { + myJpaValidationSupportChain.removeValidationSupport(validationSupport); + } + } + + /** + * Make sure that we do something sane when validating throws an unexpected exception + */ + @Test + @Ignore + public void testValidate_TermSvcHasDatabaseRollback() { + BaseTermReadSvcImpl.setInvokeOnNextCallForUnitTest(() -> { + try { + myResourceTableDao.save(new ResourceTable()); + myResourceTableDao.flush(); + } catch (Exception e) { + ourLog.info("Hit expected exception: {}", e.toString()); + } + }); + + Observation obs = new Observation(); + obs.getText().setDivAsString("
Hello
"); + obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs"); + obs.setSubject(new Reference("Patient/123")); + obs.addPerformer(new Reference("Practitioner/123")); + obs.setEffective(DateTimeType.now()); + obs.setStatus(ObservationStatus.FINAL); + obs.setValue(new StringType("This is the value")); + + OperationOutcome oo; + + // Valid code + obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED); + obs.getCode().getCodingFirstRep().setSystem("http://loinc.org").setCode("CODE3").setDisplay("Display 3"); + oo = validateAndReturnOutcome(obs); + assertEquals(encode(oo), "No issues detected during validation", oo.getIssueFirstRep().getDiagnostics()); + + } + + /** + * Make sure that we do something sane when validating throws an unexpected exception + */ + @Test + public void testValidate_TermSvcHasNpe() { + BaseTermReadSvcImpl.setInvokeOnNextCallForUnitTest(() -> { + throw new NullPointerException("MY ERROR"); + }); + + Observation obs = new Observation(); + obs.getText().setDivAsString("
Hello
"); + obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs"); + obs.setSubject(new Reference("Patient/123")); + obs.addPerformer(new Reference("Practitioner/123")); + obs.setEffective(DateTimeType.now()); + obs.setStatus(ObservationStatus.FINAL); + obs.setValue(new StringType("This is the value")); + + OperationOutcome oo; + + // Valid code + obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED); + obs.getCode().getCodingFirstRep().setSystem("http://loinc.org").setCode("CODE99999").setDisplay("Display 3"); + try { + validateAndReturnOutcome(obs); + fail(); + } catch (NullPointerException e) { + assertEquals("MY ERROR", e.getMessage()); + } + + } + @Test public void testValidateCodeableConceptWithNoSystem() { AllergyIntolerance allergy = new AllergyIntolerance(); @@ -605,6 +741,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { myDaoConfig.setAllowExternalReferences(new DaoConfig().isAllowExternalReferences()); myDaoConfig.setMaximumExpansionSize(DaoConfig.DEFAULT_MAX_EXPANSION_SIZE); myDaoConfig.setPreExpandValueSets(new DaoConfig().isPreExpandValueSets()); + + BaseTermReadSvcImpl.setInvokeOnNextCallForUnitTest(null); } @Test diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerTestDstu3.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerDstu3Test.java similarity index 94% rename from hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerTestDstu3.java rename to hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerDstu3Test.java index 3924b5d27af..15d5605414b 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerTestDstu3.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/IgInstallerDstu3Test.java @@ -4,6 +4,7 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao; import ca.uhn.fhir.jpa.dao.dstu3.BaseJpaDstu3Test; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import ca.uhn.fhir.test.utilities.JettyUtil; import ca.uhn.fhir.test.utilities.ProxyUtil; import org.eclipse.jetty.server.Server; @@ -30,9 +31,9 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class IgInstallerTestDstu3 extends BaseJpaDstu3Test { +public class IgInstallerDstu3Test extends BaseJpaDstu3Test { - private static final Logger ourLog = LoggerFactory.getLogger(IgInstallerTestDstu3.class); + private static final Logger ourLog = LoggerFactory.getLogger(IgInstallerDstu3Test.class); @Autowired private DaoConfig daoConfig; @Autowired @@ -185,7 +186,7 @@ public class IgInstallerTestDstu3 extends BaseJpaDstu3Test { ); fail(); } catch (InvalidRequestException e) { - assertEquals("", e.getMessage()); + assertEquals("Package ID nictiz.fhir.nl.stu3.questionnaires doesn't match expected: blah", e.getMessage()); } } @@ -199,8 +200,8 @@ public class IgInstallerTestDstu3 extends BaseJpaDstu3Test { .setPackageUrl("http://localhost:" + myPort + "/foo.tgz") ); fail(); - } catch (InvalidRequestException e) { - assertEquals("", e.getMessage()); + } catch (ResourceNotFoundException e) { + assertEquals("Received HTTP 404 from URL: http://localhost:" + myPort + "/foo.tgz", e.getMessage()); } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/JpaPackageCacheTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/JpaPackageCacheTest.java index c71958e63bf..b3db84a4a68 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/JpaPackageCacheTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/packages/JpaPackageCacheTest.java @@ -19,7 +19,7 @@ public class JpaPackageCacheTest extends BaseJpaR4Test { @Test public void testSavePackage() throws IOException { - try (InputStream stream = IgInstallerTestDstu3.class.getResourceAsStream("/packages/basisprofil.de.tar.gz")) { + try (InputStream stream = IgInstallerDstu3Test.class.getResourceAsStream("/packages/basisprofil.de.tar.gz")) { myPackageCacheManager.addPackageToCache("basisprofil.de", "0.2.40", stream, "basisprofil.de"); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/util/JpaClasspathTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/util/JpaClasspathTest.java new file mode 100644 index 00000000000..0bb692cab7d --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/util/JpaClasspathTest.java @@ -0,0 +1,25 @@ +package ca.uhn.fhir.jpa.util; + +import org.junit.Test; + +import static org.junit.Assert.fail; + +public class JpaClasspathTest { + + /** + * Make sure no dependencies start bringing in log4j - This makes hibernate decide to start using log4j instead of + * slf4j which is super annoying.. + */ + @Test + public void testNoLog4jOnClasspath() { + + try { + Class.forName("org.apache.logging.log4j.status.StatusLogger"); + fail("org.apache.logging.log4j.status.StatusLogger" + " found on classpath - Make sure log4j isn't being introduced"); + } catch (ClassNotFoundException theE) { + // good + } + + } + +} diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java index 5b7975af090..3076d31826d 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java @@ -122,6 +122,14 @@ public class ValidationSupportChain implements IValidationSupport { myChain.add(theIndex, theValidationSupport); } + /** + * Removes an item from the chain. Note that this method is mostly intended for testing. Removing items from the chain while validation is + * actually occurring is not an expected use case for this class. + */ + public void removeValidationSupport(IValidationSupport theValidationSupport) { + myChain.remove(theValidationSupport); + } + @Override public ValueSetExpansionOutcome expandValueSet(ValidationSupportContext theValidationSupportContext, ValueSetExpansionOptions theExpansionOptions, IBaseResource theValueSetToExpand) { for (IValidationSupport next : myChain) { diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java index 2ad279ce0bd..41d5b44f6e6 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java @@ -309,7 +309,9 @@ public class FhirInstanceValidator extends BaseValidatorBridge implements IInsta */ public void invalidateCaches() { myValidationSupport.invalidateCaches(); - myWrappedWorkerContext.invalidateCaches(); + if (myWrappedWorkerContext != null) { + myWrappedWorkerContext.invalidateCaches(); + } } From 4119cbb0c48609e0d61eba450c99cd3b36e4bf12 Mon Sep 17 00:00:00 2001 From: jamesagnew Date: Wed, 17 Jun 2020 18:28:18 -0400 Subject: [PATCH 08/22] Terser should create correct Enumeration on create --- .../src/main/java/ca/uhn/fhir/util/FhirTerser.java | 9 ++++++++- .../test/java/ca/uhn/fhir/util/FhirTerserR4Test.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java index 8debeba2b3a..fa7306678c1 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/FhirTerser.java @@ -454,7 +454,14 @@ public class FhirTerser { List values = nextDef.getAccessor().getValues(theCurrentObj); if (values.isEmpty() && theCreate) { - IBase value = nextDef.getChildByName(name).newInstance(); + BaseRuntimeElementDefinition childByName = nextDef.getChildByName(name); + Object arg = nextDef.getInstanceConstructorArguments(); + IBase value; + if (arg != null) { + value = childByName.newInstance(arg); + } else { + value = childByName.newInstance(); + } nextDef.getMutator().addValue(theCurrentObj, value); List list = new ArrayList<>(); list.add(value); diff --git a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java index 2defc5edab3..c06e09405c0 100644 --- a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java +++ b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.parser.DataFormatException; import org.hamcrest.Matchers; import org.hl7.fhir.instance.model.api.*; import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.Enumeration; import org.hl7.fhir.r4.model.Patient.LinkType; import org.junit.AfterClass; import org.junit.Assert; @@ -31,6 +32,15 @@ public class FhirTerserR4Test { private static final Logger ourLog = LoggerFactory.getLogger(FhirTerserR4Test.class); private static FhirContext ourCtx = FhirContext.forR4(); + @Test + public void testGetValuesCreateEnumeration_SetsEnumFactory() { + + Patient patient = new Patient(); + + Enumeration enumeration = (Enumeration) ourCtx.newTerser().getValues(patient, "Patient.gender", Enumeration.class, true).get(0); + assertNotNull(enumeration.getEnumFactory()); + } + @Test public void testClear() { Bundle input = new Bundle(); From 46c00f4efbe983f0cee7e3a403266063af15836b Mon Sep 17 00:00:00 2001 From: James Agnew Date: Thu, 18 Jun 2020 16:55:11 -0400 Subject: [PATCH 09/22] Update for Loinc 2.68 (#1917) * Work on loinc updates * Work on loinc upload * Clean up CLI config * Add changelog * Update hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermLoaderSvcImpl.java Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com> * Update hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermLoaderSvcImpl.java Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com> * Update hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/TerminologyUploaderProviderR4Test.java Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com> * Update hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com> * Update hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com> * Clean up imports * LOINC fixes * Loinc loader fixes Co-authored-by: IanMMarshall <49525404+IanMMarshall@users.noreply.github.com> --- .../ca/uhn/fhir/jpa/demo/CommonConfig.java | 60 +- .../5_1_0/1917-account-for-loinc268.yaml | 4 + .../dao/r4/FhirResourceDaoCodeSystemR4.java | 4 +- .../ca/uhn/fhir/jpa/entity/TermConcept.java | 4 +- .../fhir/jpa/entity/TermConceptProperty.java | 75 +- .../uhn/fhir/jpa/term/TermLoaderSvcImpl.java | 24 +- .../uhn/fhir/jpa/term/loinc/LoincHandler.java | 39 +- .../jpa/term/loinc/LoincPartLinkHandler.java | 62 +- .../LoincPartRelatedCodeMappingHandler.java | 3 + .../term/loinc/LoincUploadPropertiesEnum.java | 8 +- .../ca/uhn/fhir/jpa/term/loinc/loinc.xml | 960 ++++++++++-------- .../jpa/term/loinc/loincupload.properties | 83 ++ .../r4/TerminologyUploaderProviderR4Test.java | 3 +- ...minologyLoaderSvcIntegrationDstu3Test.java | 12 +- .../term/TerminologyLoaderSvcLoincTest.java | 20 +- .../AccessoryFiles/PartFile/LoincPartLink.csv | 10 - .../PartFile/LoincPartLink_Primary.csv | 7 + .../PartFile/LoincPartLink_Supplementary.csv | 13 + .../resources/loinc/loincupload.properties | 19 +- .../ca/uhn/fhir/util/FhirTerserR4Test.java | 39 +- 20 files changed, 835 insertions(+), 614 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1917-account-for-loinc268.yaml create mode 100644 hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loincupload.properties delete mode 100644 hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv create mode 100644 hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Primary.csv create mode 100644 hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java index 859a73641e3..86e03ec3d15 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java @@ -20,28 +20,18 @@ package ca.uhn.fhir.jpa.demo; * #L% */ -import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.model.config.PartitionSettings; import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory; -import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.lang3.time.DateUtils; import org.hibernate.dialect.H2Dialect; -import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; -import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; -import pl.allegro.tech.embeddedelasticsearch.PopularProperties; -import javax.annotation.PreDestroy; import javax.sql.DataSource; -import java.io.IOException; import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -110,55 +100,7 @@ public class CommonConfig { extraProperties.put("hibernate.search.autoregister_listeners", "false"); } - return configureElasticearch(extraProperties); - } - - private Properties configureElasticearch(Properties theExtraProperties) { - - String elasticsearchHost = "localhost"; - String elasticsearchUserId = ""; - String elasticsearchPassword = ""; - int elasticsearchPort = embeddedElasticSearch().getHttpPort(); - - new ElasticsearchHibernatePropertiesBuilder() - .setDebugRefreshAfterWrite(true) - .setDebugPrettyPrintJsonLog(true) - .setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE) - .setIndexManagementWaitTimeoutMillis(10000) - .setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW) - .setRestUrl("http://" + elasticsearchHost + ":" + elasticsearchPort) - .setUsername(elasticsearchUserId) - .setPassword(elasticsearchPassword) - .apply(theExtraProperties); - - return theExtraProperties; - - } - - @Bean - public EmbeddedElastic embeddedElasticSearch() { - String ELASTIC_VERSION = "6.5.4"; - - EmbeddedElastic embeddedElastic = null; - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) - .withSetting(PopularProperties.HTTP_PORT, 0) - .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) - .withStartTimeout(60, TimeUnit.SECONDS) - .build() - .start(); - } catch (IOException | InterruptedException e) { - throw new ConfigurationException(e); - } - - return embeddedElastic; - } - - @PreDestroy - public void stop() { - embeddedElasticSearch().stop(); + return extraProperties; } @Bean diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1917-account-for-loinc268.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1917-account-for-loinc268.yaml new file mode 100644 index 00000000000..199e3832485 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1917-account-for-loinc268.yaml @@ -0,0 +1,4 @@ +--- +type: add +issue: 1917 +title: "The LOINC importer has been updated to support the file format used by the LOINC 2.68 release." diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoCodeSystemR4.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoCodeSystemR4.java index a936c87792f..f2c7f221e30 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoCodeSystemR4.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoCodeSystemR4.java @@ -105,11 +105,11 @@ public class FhirResourceDaoCodeSystemR4 extends BaseHapiFhirResourceDao theFiles, RequestDetails theRequestDetails) { try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) { - List loincUploadPropertiesFragment = Collections.singletonList( - LOINC_UPLOAD_PROPERTIES_FILE.getCode() - ); - descriptors.verifyMandatoryFilesExist(loincUploadPropertiesFragment); - Properties uploadProperties = getProperties(descriptors, LOINC_UPLOAD_PROPERTIES_FILE.getCode()); List mandatoryFilenameFragments = Arrays.asList( @@ -119,7 +114,8 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { uploadProperties.getProperty(LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE.getCode(), LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_IMAGING_DOCUMENT_CODES_FILE.getCode(), LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_PART_FILE.getCode(), LOINC_PART_FILE_DEFAULT.getCode()), - uploadProperties.getProperty(LOINC_PART_LINK_FILE.getCode(), LOINC_PART_LINK_FILE_DEFAULT.getCode()), + uploadProperties.getProperty(LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()), + uploadProperties.getProperty(LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(), LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_PART_RELATED_CODE_MAPPING_FILE.getCode(), LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_RSNA_PLAYBOOK_FILE.getCode(), LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()), uploadProperties.getProperty(LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE.getCode(), LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT.getCode()), @@ -238,6 +234,13 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { @NotNull private Properties getProperties(LoadedFileDescriptors theDescriptors, String thePropertiesFile) { Properties retVal = new Properties(); + + try (InputStream propertyStream = TermLoaderSvcImpl.class.getResourceAsStream("/ca/uhn/fhir/jpa/term/loinc/loincupload.properties")) { + retVal.load(propertyStream); + } catch (IOException e) { + throw new InternalErrorException("Failed to process loinc.properties", e); + } + for (FileDescriptor next : theDescriptors.getUncompressedFileDescriptors()) { if (next.getFilename().endsWith(thePropertiesFile)) { try { @@ -425,10 +428,6 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { handler = new LoincRsnaPlaybookHandler(code2concept, valueSets, conceptMaps, theUploadProperties); iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_RSNA_PLAYBOOK_FILE.getCode(), LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); - // Part link - handler = new LoincPartLinkHandler(codeSystemVersion, code2concept); - iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE.getCode(), LOINC_PART_LINK_FILE_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); - // Part related code mapping handler = new LoincPartRelatedCodeMappingHandler(code2concept, valueSets, conceptMaps, theUploadProperties); iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PART_RELATED_CODE_MAPPING_FILE.getCode(), LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); @@ -469,6 +468,11 @@ public class TermLoaderSvcImpl implements ITermLoaderSvc { handler = new LoincParentGroupFileHandler(code2concept, valueSets, conceptMaps, theUploadProperties); iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PARENT_GROUP_FILE.getCode(), LOINC_PARENT_GROUP_FILE_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); + // Part link + handler = new LoincPartLinkHandler(codeSystemVersion, code2concept, propertyNamesToTypes); + iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); + iterateOverZipFile(theDescriptors, theUploadProperties.getProperty(LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(), LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()), handler, ',', QuoteMode.NON_NUMERIC, false); + IOUtils.closeQuietly(theDescriptors); valueSets.add(getValueSetLoincAll()); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java index ecf3dce5c73..8f9fad210bc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincHandler.java @@ -86,44 +86,9 @@ public class LoincHandler implements IRecordHandler { concept.addPropertyString(nextPropertyName, nextPropertyValue); break; case CODING: - // TODO: handle "Ser/Plas^Donor" - String propertyValue = nextPropertyValue; - if (nextPropertyName.equals("COMPONENT")) { - if (propertyValue.contains("^")) { - propertyValue = propertyValue.substring(0, propertyValue.indexOf("^")); - } else if (propertyValue.contains("/")) { - propertyValue = propertyValue.substring(0, propertyValue.indexOf("/")); - } - } - - PartTypeAndPartName key = new PartTypeAndPartName(nextPropertyName, propertyValue); - String partNumber = myPartTypeAndPartNameToPartNumber.get(key); - - if (partNumber == null && nextPropertyName.equals("TIME_ASPCT")) { - key = new PartTypeAndPartName("TIME", nextPropertyValue); - partNumber = myPartTypeAndPartNameToPartNumber.get(key); - } - if (partNumber == null && nextPropertyName.equals("METHOD_TYP")) { - key = new PartTypeAndPartName("METHOD", nextPropertyValue); - partNumber = myPartTypeAndPartNameToPartNumber.get(key); - } - if (partNumber == null && nextPropertyName.equals("SCALE_TYP")) { - key = new PartTypeAndPartName("SCALE", nextPropertyValue); - partNumber = myPartTypeAndPartNameToPartNumber.get(key); - } - - if (partNumber == null && nextPropertyName.equals("SYSTEM") && nextPropertyValue.startsWith("^")) { - continue; - } - - if (isNotBlank(partNumber)) { - concept.addPropertyCoding(nextPropertyName, ITermLoaderSvc.LOINC_URI, partNumber, nextPropertyValue); - } else { - String msg = "Unable to find part code with TYPE[" + key.getPartType() + "] and NAME[" + nextPropertyValue + "] (using name " + propertyValue + ")"; - ourLog.warn(msg); -// throw new InternalErrorException(msg); - } + // These are handles by the LOINC PartLink file break; + case DECIMAL: case CODE: case INTEGER: diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartLinkHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartLinkHandler.java index e9e1bae1031..ccce7446de9 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartLinkHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartLinkHandler.java @@ -22,12 +22,17 @@ package ca.uhn.fhir.jpa.term.loinc; import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.entity.TermConceptProperty; import ca.uhn.fhir.jpa.term.IRecordHandler; +import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import org.apache.commons.csv.CSVRecord; +import org.hl7.fhir.r4.model.CodeSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; +import java.util.Optional; import static org.apache.commons.lang3.StringUtils.trim; @@ -36,40 +41,65 @@ public class LoincPartLinkHandler implements IRecordHandler { private static final Logger ourLog = LoggerFactory.getLogger(LoincPartLinkHandler.class); private final Map myCode2Concept; private final TermCodeSystemVersion myCodeSystemVersion; + private final Map myPropertyNames; private Long myPartCount; - public LoincPartLinkHandler(TermCodeSystemVersion theCodeSystemVersion, Map theCode2concept) { + public LoincPartLinkHandler(TermCodeSystemVersion theCodeSystemVersion, Map theCode2concept, Map thePropertyNames) { myCodeSystemVersion = theCodeSystemVersion; myCode2Concept = theCode2concept; + myPropertyNames = thePropertyNames; } @Override public void accept(CSVRecord theRecord) { String loincNumber = trim(theRecord.get("LoincNumber")); - String longCommonName = trim(theRecord.get("LongCommonName")); + String property = trim(theRecord.get("Property")); + String partName = trim(theRecord.get("PartName")); String partNumber = trim(theRecord.get("PartNumber")); + /* + * Property has the form http://loinc.org/property/COMPONENT + * but we want just the COMPONENT part + */ + int lastSlashIdx = property.lastIndexOf("/"); + String propertyPart = property.substring(lastSlashIdx + 1); + TermConcept loincConcept = myCode2Concept.get(loincNumber); - TermConcept partConcept = myCode2Concept.get(partNumber); - if (loincConcept == null) { - ourLog.warn("No loinc code: {}", loincNumber); - return; + throw new InternalErrorException("Unknown loinc code: " + loincNumber); } - if (partConcept == null) { - if (myPartCount == null) { - myPartCount = myCode2Concept - .keySet() - .stream() - .filter(t->t.startsWith("LP")) - .count(); - } - ourLog.debug("No part code: {} - Have {} part codes", partNumber, myPartCount); + + CodeSystem.PropertyType propertyType = myPropertyNames.get(propertyPart); + if (propertyType == null) { return; } - // For now we're ignoring these + String expectedValue; + if (propertyType == CodeSystem.PropertyType.STRING) { + expectedValue = partName; + } else if (propertyType == CodeSystem.PropertyType.CODING) { + expectedValue = partNumber; + } else { + throw new InternalErrorException("Don't know how to handle property of type: " + propertyType); + } + + Optional existingProprty = loincConcept + .getProperties() + .stream() + .filter(t -> t.getKey().equals(propertyPart)) + .filter(t -> t.getValue().equals(expectedValue)) + .findFirst(); + if (existingProprty.isPresent()) { + return; + } + + ourLog.info("Adding new property {} = {}", propertyPart, partNumber); + if (propertyType == CodeSystem.PropertyType.STRING) { + loincConcept.addPropertyString(propertyPart, partName); + } else { + loincConcept.addPropertyCoding(propertyPart, ITermLoaderSvc.LOINC_URI, partNumber, partName); + } } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartRelatedCodeMappingHandler.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartRelatedCodeMappingHandler.java index 02e08bc533e..5fefd31a549 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartRelatedCodeMappingHandler.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincPartRelatedCodeMappingHandler.java @@ -88,6 +88,9 @@ public class LoincPartRelatedCodeMappingHandler extends BaseLoincHandler impleme case "wider": equivalence = Enumerations.ConceptMapEquivalence.WIDER; break; + case "relatedto": + equivalence = Enumerations.ConceptMapEquivalence.RELATEDTO; + break; default: throw new InternalErrorException("Unknown equivalence '" + mapType + "' for PartNumber: " + partNumber); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java index 40c36d6eb3e..d1d1295c9e2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/loinc/LoincUploadPropertiesEnum.java @@ -67,9 +67,13 @@ public enum LoincUploadPropertiesEnum { // Part LOINC_PART_FILE("loinc.part.file"), LOINC_PART_FILE_DEFAULT("AccessoryFiles/PartFile/Part.csv"), + // Part link - LOINC_PART_LINK_FILE("loinc.part.link.file"), - LOINC_PART_LINK_FILE_DEFAULT("AccessoryFiles/PartFile/LoincPartLink.csv"), + LOINC_PART_LINK_FILE_PRIMARY("loinc.part.link.primary.file"), + LOINC_PART_LINK_FILE_PRIMARY_DEFAULT("AccessoryFiles/PartFile/LoincPartLink_Primary.csv"), + LOINC_PART_LINK_FILE_SUPPLEMENTARY("loinc.part.link.supplementary.file"), + LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT("AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv"), + // Part related code mapping LOINC_PART_RELATED_CODE_MAPPING_FILE("loinc.part.related.code.mapping.file"), LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT("AccessoryFiles/PartFile/PartRelatedCodeMapping.csv"), diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loinc.xml b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loinc.xml index 7fc6b37e371..efe4184e99b 100644 --- a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loinc.xml +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loinc.xml @@ -1,442 +1,548 @@ - + - - - - - - - - - - - - + + + + + + + + + + + + + + <status value="active"/> + <experimental value="false"/> + + <publisher value="Regenstrief Institute, Inc."/> + <contact> + <telecom> + <value value="http://loinc.org"/> + </telecom> + </contact> + + <!-- + <date value=2020-06/> + --> + <description value="LOINC is a freely available international standard for tests, measurements, and observations"/> + <copyright value="This material contains content from LOINC (http://loinc.org). LOINC is copyright ©1995-2020, Regenstrief Institute, Inc. and the Logical Observation Identifiers Names and Codes (LOINC) Committee and is available at no cost under the license at http://loinc.org/license. LOINC® is a registered United States trademark of Regenstrief Institute, Inc."/> + <caseSensitive value="false"/> + + <valueSet value=" http://loinc.org/vs"/> + <!-- + For a version specific reference: + <valueSet value="http://loinc.org/2.68/vs"/> --> - <!-- if a specific version is specified, the name should carry this information should be in the name (e.g. LOINC_259) and title --> - <name value="LOINC"/> - <title value="LOINC Code System"/> - <status value="active"/> - <experimental value="false"/> - - <publisher value="Regenstrief Institute, Inc."/> - <contact> - <telecom> - <value value="http://loinc.org"/> - </telecom> - </contact> - - <!-- - <date value=[date for this version]"/> - --> - <description value="LOINC is a freely available international standard for tests, measurements, and observations"/> - <copyright value="This content from LOINC® is copyright © 1995 Regenstrief Institute, Inc. and the LOINC Committee, and available at no cost under the license at http://loinc.org/terms-of-use"/> - <caseSensitive value="false"/> - - <valueSet value=" http://loinc.org/vs"/> - <!-- - for a version specific reference: - <valueSet value="http://loinc.org/2.56/vs"/> - --> - - <!-- - It's at the discretion of servers whether to present fragments of LOINC heirarchically or not, when - using the code system resource. But, if they are heirarchical, the Hierarchy SHALL be based on the is-a relationship that is derived from the LOINC Multiaxial Hierarchy. - --> - <HierarchyMeaning value="is-a"/> - <compositional value="false"/> <!-- no compositional grammar in LOINC --> - <versionNeeded value="false"/> - - <!-- this canonical definition of LOINC does not include the content. - Servers may choose to include fragments (but not, due to size constraints, all of LOINC) --> - <content value="not-present"/> - -<!-- <count value="65000"/>... if working with a specific version, you could nominate a count of the total number of concepts (including the answers, Hierarchy, etc.) --> + <!-- + It's at the discretion of servers whether to present fragments of LOINC hierarchically or not, when using the code system resource. But, if they are hierarchical, the Hierarchy SHALL be based on the is-a relationship that is derived from the LOINC Multiaxial Hierarchy. + --> + <hierarchyMeaning value="is-a"/> + <compositional value="false"/> <!-- no compositional grammar in LOINC --> + <versionNeeded value="false"/> - <!-- - Generally defined filters for specifying value sets - In LOINC, all the properties can be used as filters too, but they are not defined explicitly as filters as well. - Note that parent/child/ancestor/descendant are defined by FHIR, but repeated here to document them clearly. - - For illustration purposes, consider this slice of the LOINC Multiaxial Hierarchy when reading the descriptions: - - Microbiology [LP31755-9] - Microorganism [LP14559-6] - Virus [LP14855-8] - Zika virus [LP200137-0] - Zika virus RNA | XXX [LP203413-2] - Zika virus RNA [Presence] in Unspecified specimen by Probe and target amplification method [79190-5] + <!-- + This canonical definition of LOINC does not include the LOINC content, which is distributed separately for portability. - Language Note: The filters defined here are specified using the default LOINC language - English (US). Requests are meant to be specified and interpreted on the English version. The return can be in a specified language (if supported by the server). But note that not all filters/properties have language translations available. - --> - <filter> - <code value="parent"/> - <description value="Allows for the selection of a set of codes based on their appearance in the LOINC Multiaxial Hierarchy. parent selects immediate children only. For example, the code '79190-5' has the parent 'LP203413-2'"/> - <operator value="="/> - <value value="A Part code"/> - </filter> - <filter> - <code value="child"/> - <description value="Allows for the selection of a set of codes based on their appearance in the LOINC Multiaxial Hierarchy. child selects immediate children only. For example, the code 'LP203413-2' has the child '79190-5'"/> - <operator value="in"/> - <value value="A comma separated list of Part codes"/> - </filter> - <filter> - <code value="ancestor"/> - <description value="Allows for the selection of a set of codes based on their appearance in the LOINC Multiaxial Hierarchy. ancestor includes parents transitively, e.g. 'LP203413-2' eventually has an ancestor 'LP14559-6', so the code '79190-5' is in the set of codes that have ancestor=LP14559-6"/> - <operator value="="/> - <value value="A Part code"/> - </filter> - <filter> - <code value="descendant"/> - <description value="Allows for the selection of a set of codes based on their appearance in the LOINC Multiaxial Hierarchy. descendant includes children transitively, e.g. 'LP14559-6' eventually has a descendant 'LP203413-2', so the code '79190-5' is in the set of codes that have descendant=LP14559-6"/> - <operator value="in"/> - <value value="A comma separated list of Part codes"/> - </filter> - <filter> - <code value="copyright"/> - <description value="Allows for the inclusion or exclusion of LOINC codes that include 3rd party copyright notices. LOINC = only codes with a sole copyright by Regenstrief. 3rdParty = only codes with a 3rd party copyright in addition to the one from Regenstrief"/> - <operator value="="/> - <value value="LOINC | 3rdParty"/> - </filter> - <!-- properties. There are 3 kinds of properties: - fhir: display, designation; these are not described here since they are inherent in the specification - infrastructural: defined by FHIR, but documented here for LOINC - LOINC properties: defined by the main LOINC table - concept model: defined by the LOINC Multiaxial Hierarchy - --> - <!-- first, the infrastructural properties - inherited from FHIR, but documented here --> - <property> - <code value="parent"/> - <uri value="http://hl7.org/fhir/concept-properties#parent"/> - <description value="A parent code in the Multiaxial Hierarchy"/> - <type value=""/> - </property> - <property> - <code value="child"/> - <uri value="http://hl7.org/fhir/concept-properties#child"/> - <description value="A child code in the Multiaxial Hierarchy"/> - <type value=""/> - </property> - <!-- - LOINC properties. - These apply to the main LOINC codes, but not the Multiaxial Hierarchy, the answer lists, or the part codes. - - Notes: - SHORTNAME = display & LONG_COMMON_NAME = definition - Properties are specified as type "code", which are LOINC Part codes (LP-). - It is anticipated that the LOINC Part codes to be used in these properties will be published in the June 2017 LOINC release. - --> - <property> - <code value="STATUS"/> - <uri value="http://loinc.org/property/STATUS"/> - <description value="Status of the term. Within LOINC, codes with STATUS=DEPRECATED are considered inactive. Current values: ACTIVE, TRIAL, DISCOURAGED, and DEPRECATED"/> - <!-- DV NOTE: changed this from boolean to string --> - <type value="string"/> - </property> - <property> - <code value="COMPONENT"/> - <uri value="http://loinc.org/property/COMPONENT"/> - <description value="First major axis-component or analyte: Analyte Name, Analyte sub-class, Challenge"/> - <type value="Coding"/> - </property> - <property> - <code value="PROPERTY"/> - <uri value="http://loinc.org/property/PROPERTY"/> - <description value="Second major axis-property observed: Kind of Property (also called kind of quantity)"/> - <type value="Coding"/> - </property> - <property> - <code value="TIME_ASPCT"/> - <uri value="http://loinc.org/property/TIME_ASPCT"/> - <description value="Third major axis-timing of the measurement: Time Aspect (Point or moment in time vs. time interval)"/> - <type value="Coding"/> - </property> - <property> - <code value="SYSTEM"/> - <uri value="http://loinc.org/property/SYSTEM"/> - <description value="Fourth major axis-type of specimen or system: System (Sample) Type"/> - <type value="Coding"/> - </property> - <property> - <code value="SCALE_TYP"/> - <uri value="http://loinc.org/property/SCALE_TYP"/> - <description value="Fifth major axis-scale of measurement: Type of Scale"/> - <type value="Coding"/> - </property> - <property> - <code value="METHOD_TYP"/> - <uri value="http://loinc.org/property/METHOD_TYP"/> - <description value="Sixth major axis-method of measurement: Type of Method"/> - <type value="Coding"/> - </property> - <property> - <code value="CLASS"/> - <uri value="http://loinc.org/property/CLASS"/> - <description value="An arbitrary classification of the terms for grouping related observations together"/> - <type value="string"/> - </property> - <!-- Note: removed in 0.3 - <property> - <code value="CHNG_TYPE"/> - <uri value="http://loinc.org/property/CHNG_TYPE"/> - <description value="A classification of the type of change made to a LOINC term, e.g. DEL=deprecated, ADD=add"/> - <type value="string"/> - </property> - --> - <property> - <code value="VersionLastChanged"/> - <uri value="http://loinc.org/property/VersionLastChanged"/> - <description value="The LOINC version number in which the record has last changed. For new records, this field contains the same value as the FirstPublishedRelease property."/> - <type value="string"/> - </property> - <property> - <code value="CONSUMER_NAME"/> - <uri value="http://loinc.org/property/CONSUMER_NAME"/> - <description value="An experimental (beta) consumer friendly name for this item. The intent is to provide a test name that health care consumers will recognize; it will be similar to the names that might appear on a lab report"/> - <type value="string"/> - </property> - <property> - <code value="CLASSTYPE"/> - <uri value="http://loinc.org/property/CLASSTYPE"/> - <description value="1=Laboratory class; 2=Clinical class; 3=Claims attachments; 4=Surveys"/> - <type value="string"/> - </property> - <property> - <code value="ORDER_OBS"/> - <uri value="http://loinc.org/property/ORDER_OBS"/> - <description value="Provides users with an idea of the intended use of the term by categorizing it as an order only, observation only, or both"/> - <type value="string"/> - </property> - <property> - <code value="HL7_ATTACHMENT_STRUCTURE"/> - <uri value="http://loinc.org/property/HL7_ATTACHMENT_STRUCTURE"/> - <description value="This property is populated in collaboration with the HL7 Attachments Work Group as described in the HL7 Attachment Specification: Supplement to Consolidated CDA Templated Guide."/> - <type value="string"/> - </property> - <property> - <code value="VersionFirstReleased"/> - <uri value="http://loinc.org/property/VersionFirstReleased"/> - <description value="This is the LOINC version number in which this LOINC term was first published."/> - <type value="string"/> - </property> - <property> - <code value="PanelType"/> - <uri value="http://loinc.org/property/PanelType"/> - <description value="For LOINC terms that are panels, this attribute classifies them as a 'Convenience group', 'Organizer', or 'Panel'"/> - <type value="string"/> - </property> - <property> - <code value="ValidHL7AttachmentRequest"/> - <uri value="http://loinc.org/property/ValidHL7AttachmentRequest"/> - <description value="A value of Y in this field indicates that this LOINC code can be sent by a payer as part of an HL7 Attachment request for additional information."/> - <type value="string"/> - </property> + Servers may choose to include fragments of LOINC for illustration purposes. + --> + <content value="not-present"/> - <!-- LOINC/RSNA Radiology Playbook properties. These apply only to terms in the LOINC/RSNA Radiology Playbook File. - Notes: - Properties are specified as type "code", which are LOINC Part codes (LP-) - Converted the attribute names from LOINC style to FHIR style b/c they contained periods - Maneuver sub-attributes are being released in 2016 12. - --> - <property> - <code value="rad-modality-modality-type"/> - <uri value="http://loinc.org/property/rad-modality-type"/> - <description value="Modality is used to represent the device used to acquire imaging information."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-modality-modality-subtype"/> - <uri value="http://loinc.org/property/rad-modality-subtype"/> - <description value="Modality subtype may be optionally included to signify a particularly common or evocative configuration of the modality."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-anatomic-location-region-imaged"/> - <uri value="http://loinc.org/property/rad-anatomic-location-region-imaged"/> - <description value="The Anatomic Location Region Imaged attribute is used in two ways: as a coarse-grained descriptor of the area imaged and a grouper for finding related imaging exams; or, it is used just as a grouper."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-anatomic-location-imaging-focus"/> - <uri value="http://loinc.org/property/rad-anatomic-location-imaging-focus"/> - <description value="The Anatomic Location Imaging Focus is a more fine-grained descriptor of the specific target structure of an imaging exam. In many areas, the focus should be a specific organ."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-anatomic-location-laterality-presence"/> - <uri value="http://loinc.org/property/rad-anatomic-location-laterality-presence"/> - <description value="Radiology Exams that require laterality to be specified in order to be performed are signified with an Anatomic Location Laterality Presence attribute set to 'True'"/> - <type value="Coding"/> - </property> - <property> - <code value="rad-anatomic-location-laterality"/> - <uri value="http://loinc.org/property/rad-anatomic-location-laterality"/> - <description value="Radiology exam Laterality is specified as one of: Left, Right, Bilateral, Unilateral, Unspecified"/> - <type value="Coding"/> - </property> - <property> - <code value="rad-view-view-aggregation"/> - <uri value="http://loinc.org/property/rad-view-aggregation"/> - <description value="Aggregation describes the extent of the imaging performed, whether in quantitative terms (e.g., '3 or more views') or subjective terms (e.g., 'complete')."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-view-view-type"/> - <uri value="http://loinc.org/property/rad-view-view-type"/> - <description value="View type names specific views, such as 'lateral' or 'AP'."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-maneuver-maneuver-type"/> - <uri value="http://loinc.org/property/rad-maneuver-maneuver-type"/> - <description value="Maneuver type indicates an action taken with the goal of elucidating or testing a dynamic aspect of the anatomy."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-timing"/> - <uri value="http://loinc.org/property/rad-timing"/> - <description value="The Timing/Existence property used in conjunction with pharmaceutical and manueuver properties. It specifies whether or not the imaging occurs in the presence of the administered pharmaceutical or a manuever designed to test some dynamic aspect of anatomy or physiology ."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-pharmaceutical-substance-given"/> - <uri value="http://loinc.org/property/rad-pharmaceutical-substance-given"/> - <description value="The Pharmaceutical Substance Given specifies administered contrast agents, radiopharmaceuticals, medications, or other clinically important agents and challenges during the imaging procedure."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-pharmaceutical-route"/> - <uri value="http://loinc.org/property/rad-pharmaceutical-route"/> - <description value="Route specifies the route of administration of the pharmeceutical."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-reason-for-exam"/> - <uri value="http://loinc.org/property/rad-reason-for-exam"/> - <description value="Reason for exam is used to describe a clinical indication or a purpose for the study."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-guidance-for-presence"/> - <uri value="http://loinc.org/property/rad-guidance-for-presence"/> - <description value="Guidance for.Presence indicates when a procedure is guided by imaging."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-guidance-for-approach"/> - <uri value="http://loinc.org/property/rad-guidance-for-approach"/> - <description value="Guidance for.Approach refers to the primary route of access used, such as percutaneous, transcatheter, or transhepatic."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-guidance-for-action"/> - <uri value="http://loinc.org/property/rad-guidance-for-action"/> - <description value="Guidance for.Action indicates the intervention performed, such as biopsy, aspiration, or ablation."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-guidance-for-object"/> - <uri value="http://loinc.org/property/rad-guidance-for-object"/> - <description value="Guidance for.Object specifies the target of the action, such as mass, abscess or cyst."/> - <type value="Coding"/> - </property> - <property> - <code value="rad-subject"/> - <uri value="http://loinc.org/property/rad-subject"/> - <description value="Subject is intended for use when there is a need to distinguish between the patient associated with an imaging study, and the target of the study."/> - <type value="Coding"/> - </property> - <!-- Document Ontology properties. These apply only to terms in the LOINC Document Ontology File - Notes: - Properties are specified as type "code", which are LOINC Part codes (LP-) - Converted the attribute names from LOINC style to FHIR style b/c they contained periods - --> - <property> - <code value="document-kind"/> - <uri value="http://loinc.org/property/document-kind"/> - <description value="Characterizes the general structure of the document at a macro level."/> - <type value="Coding"/> - </property> - <property> - <code value="document-role"/> - <uri value="http://loinc.org/property/document-role"/> - <description value="Characterizes the training or professional level of the author of the document, but does not break down to specialty or subspecialty.."/> - <type value="Coding"/> - </property> - <property> - <code value="document-setting"/> - <uri value="http://loinc.org/property/document-setting"/> - <description value="Setting is a modest extension of CMS’s coarse definition of care settings, such as outpatient, hospital, etc. Setting is not equivalent to location, which typically has more locally defined meanings."/> - <type value="Coding"/> - </property> - <property> - <code value="document-subject-matter-domain"/> - <uri value="http://loinc.org/property/document-subject-matter-domain"/> - <description value="Characterizes the clinical domain that is the subject of the document. For example, Internal Medicine, Neurology, Physical Therapy, etc."/> - <type value="Coding"/> - </property> - <property> - <code value="document-type-of-service"/> - <uri value="http://loinc.org/property/document-type-of-service"/> - <description value="Characterizes the kind of service or activity provided to/for the patient (or other subject of the service) that is described in the document."/> - <type value="Coding"/> - </property> - <!-- Answer list related properties --> - <property> - <code value="answer-list"/> - <uri value="http://loinc.org/property/answer-list"/> - <description value="An answer list associated with this LOINC code (if there are matching answer lists defined). Only on normal LOINC Codes"/> - <type value="Coding"/> - </property> - <!-- Note: We expect to add an AnswerListType property when LOINC publishes new answer list file format in June 2017 --> - <property> - <code value="answers-for"/> - <uri value="http://loinc.org/property/answers-for"/> - <description value="A LOINC Code for which this answer list is used. Only on normal LL- Codes"/> - <type value="Coding"/> - </property> - <!-- Note for future consideration. These are properties of LA codes in the context of a particular list. Not global properties - <property> - <code value="sequence"/> - <uri value="http://loinc.org/property/sequence"/> - <description value="Sequence Number of a answer in a set of answers (LA- codes only)"/> - <type value="integer"/> - </property> - <property> - <code value="score"/> - <uri value="http://loinc.org/property/score"/> - <description value="Score assigned to an answer (LA- codes only)"/> - <type value="integer"/> - </property> - --> + <!-- + <count value="65000"/> + If working with a specific version, you could nominate a count of the total number of concepts (including the answers, Hierarchy, etc.). In this canonical definition we do not. + --> + + <!-- + FILTERS + Generally defined filters for specifying value sets + In LOINC, all the properties can also be used as filters, but they are not defined explicitly as filters. + Parent/child properties are as defined by FHIR. Note that at this time the LOINC code system resource does not support ancestor/descendant relationships. + + For illustration purposes, consider this slice of the LOINC Multiaxial Hierarchy when reading the descriptions below: + + Microbiology [LP7819-8] + Microorganism [LP14559-6] + Virus [LP14855-8] + Zika virus [LP200137-0] + Zika virus RNA | XXX [LP203271-4] + Zika virus RNA | XXX | Microbiology [LP379670-5] + Zika virus RNA [Presence] in Unspecified specimen by Probe and target amplification method [79190-5] + + Language Note: The filters defined here are specified using the default LOINC language - English (US). Requests are meant to be specified and interpreted on the English version. The return can be in a specified language (if supported by the server). But note that not all filters/properties have language translations available. + --> + <filter> + <code value="parent"/> + <description value="Allows for the selection of a set of codes based on their appearance in the LOINC Multiaxial Hierarchy. Parent selects immediate parent only. For example, the code '79190-5' has the parent 'LP379670-5'"/> + <operator value="="/> + <value value="A Part code"/> + </filter> + <filter> + <code value="child"/> + <description value="Allows for the selection of a set of codes based on their appearance in the LOINC Multiaxial Hierarchy. Child selects immediate children only. For example, the code 'LP379670-5' has the child '79190-5'. Only LOINC Parts have children; LOINC codes do not have any children because they are leaf nodes."/> + <operator value="="/> + <value value="A comma separated list of Part or LOINC codes"/> + </filter> + <filter> + <code value="copyright"/> + <description value="Allows for the inclusion or exclusion of LOINC codes that include 3rd party copyright notices. LOINC = only codes with a sole copyright by Regenstrief. 3rdParty = only codes with a 3rd party copyright in addition to the one from Regenstrief"/> + <operator value="="/> + <value value="LOINC | 3rdParty"/> + </filter> + + <!-- + PROPERTIES + There are 4 kinds of properties that apply to all LOINC codes: + 1. FHIR: display, designation; these are not described here since they are inherent in the specification + 2. Infrastructural: defined by FHIR, but documented here for the LOINC Multiaxial Hierarchy + 3. Primary LOINC properties: defined by the main LOINC table + 4. Secondary LOINC properties: defined by the LoincPartLink table + Additionally, there are 2 kinds of properties specific to Document ontology and Radiology codes, respectively: + 1. LOINC/RSNA Radiology Playbook properties + 2. Document Ontology properties + --> + <!-- + Infrastructural properties - inherited from FHIR, but documented here for the LOINC Multiaxial Hierarchy. + --> + + <property> + <code value="parent"/> + <uri value="http://hl7.org/fhir/concept-properties#parent"/> + <description value="A parent code in the Multiaxial Hierarchy"/> + <type value=""/> + </property> + <property> + <code value="child"/> + <uri value="http://hl7.org/fhir/concept-properties#child"/> + <description value="A child code in the Multiaxial Hierarchy"/> + <type value=""/> + </property> + + <!-- + Primary LOINC properties. + These apply to the main LOINC codes, but not the Multiaxial Hierarchy, Answer lists, or the Part codes. + Notes: + In the LOINC code system resource, the display element = LONG_COMMON_NAME + Many properties are specified as type "coding", which allows use of LOINC Part codes (LP-) and the display text. LOINC Parts and their associations to LOINC terms are published in the LOINC Part File. + The properties defined here follow the guidance of the LOINC Users' Manual, which states that they should be expressed with the LOINC attributes contained in the LOINC Table. Properties that are not defined in the LOINC Table use FHIR-styled names. + --> + + <property> + <code value="STATUS"/> + <uri value="http://loinc.org/property/STATUS"/> + <description value="Status of the term. Within LOINC, codes with STATUS=DEPRECATED are considered inactive. Current values: ACTIVE, TRIAL, DISCOURAGED, and DEPRECATED"/> + <type value="string"/> + </property> + <property> + <code value="COMPONENT"/> + <uri value="http://loinc.org/property/COMPONENT"/> + <description value="First major axis-component or analyte: Analyte Name, Analyte sub-class, Challenge"/> + <type value="Coding"/> + </property> + <property> + <code value="PROPERTY"/> + <uri value="http://loinc.org/property/PROPERTY"/> + <description value="Second major axis-property observed: Kind of Property (also called kind of quantity)"/> + <type value="Coding"/> + </property> + <property> + <code value="TIME_ASPCT"/> + <uri value="http://loinc.org/property/TIME_ASPCT"/> + <description value="Third major axis-timing of the measurement: Time Aspect (Point or moment in time vs. time interval)"/> + <type value="Coding"/> + </property> + <property> + <code value="SYSTEM"/> + <uri value="http://loinc.org/property/SYSTEM"/> + <description value="Fourth major axis-type of specimen or system: System (Sample) Type"/> + <type value="Coding"/> + </property> + <property> + <code value="SCALE_TYP"/> + <uri value="http://loinc.org/property/SCALE_TYP"/> + <description value="Fifth major axis-scale of measurement: Type of Scale"/> + <type value="Coding"/> + </property> + <property> + <code value="METHOD_TYP"/> + <uri value="http://loinc.org/property/METHOD_TYP"/> + <description value="Sixth major axis-method of measurement: Type of Method"/> + <type value="Coding"/> + </property> + <property> + <code value="CLASS"/> + <uri value="http://loinc.org/property/CLASS"/> + <description value="An arbitrary classification of the terms for grouping related observations together"/> + <type value="Coding"/> + </property> + <property> + <code value="VersionLastChanged"/> + <uri value="http://loinc.org/property/VersionLastChanged"/> + <description value="The LOINC version number in which the record has last changed. For new records, this field contains the same value as the VersionFirstReleased property."/> + <type value="string"/> + </property> + <property> + <code value="CLASSTYPE"/> + <uri value="http://loinc.org/property/CLASSTYPE"/> + <description value="1=Laboratory class; 2=Clinical class; 3=Claims attachments; 4=Surveys"/> + <type value="string"/> + </property> + <property> + <code value="ORDER_OBS"/> + <uri value="http://loinc.org/property/ORDER_OBS"/> + <description value="Provides users with an idea of the intended use of the term by categorizing it as an order only, observation only, or both"/> + <type value="string"/> + </property> + <property> + <code value="HL7_ATTACHMENT_STRUCTURE"/> + <uri value="http://loinc.org/property/HL7_ATTACHMENT_STRUCTURE"/> + <description value="This property is populated in collaboration with the HL7 Attachments Work Group as described in the HL7 Attachment Specification: Supplement to Consolidated CDA Templated Guide."/> + <type value="string"/> + </property> + <property> + <code value="VersionFirstReleased"/> + <uri value="http://loinc.org/property/VersionFirstReleased"/> + <description value="This is the LOINC version number in which this LOINC term was first published."/> + <type value="string"/> + </property> + <property> + <code value="PanelType"/> + <uri value="http://loinc.org/property/PanelType"/> + <description value="For LOINC terms that are panels, this attribute classifies them as a 'Convenience group', 'Organizer', or 'Panel'"/> + <type value="string"/> + </property> + <property> + <code value="ValidHL7AttachmentRequest"/> + <uri value="http://loinc.org/property/ValidHL7AttachmentRequest"/> + <description value="A value of Y in this field indicates that this LOINC code can be sent by a payer as part of an HL7 Attachment request for additional information."/> + <type value="string"/> + </property> + <property> + <code value="DisplayName"/> + <uri value="http://loinc.org/property/DisplayName"/> + <description value="A name that is more 'clinician-friendly' compared to the current LOINC Short Name, Long Common Name, and Fully Specified Name. It is created algorithmically from the manually crafted display text for each Part and is generally more concise than the Long Common Name."/> + <type value="string"/> + </property> + <property> + <code value="answer-list"/> + <uri value="http://loinc.org/property/answer-list"/> + <description value="An answer list associated with this LOINC code (if there are matching answer lists defined)."/> + <type value="Coding"/> + </property> + + <!-- + Secondary LOINC properties. + These properties also apply to the main LOINC codes, but not the Multiaxial Hierarchy, Answer lists, or the Part codes. + Notes: + These properties are defined in the LoincPartLink table. + --> + + <property> + <code value="analyte"/> + <uri value="http://loinc.org/property/analyte"/> + <description value="First sub-part of the Component, i.e., the part of the Component before the first carat"/> + <type value="Coding"/> + </property> + <property> + <code value="analyte-core"/> + <uri value="http://loinc.org/property/analyte-core"/> + <description value="The primary part of the analyte without the suffix"/> + <type value="Coding"/> + </property> + <property> + <code value="analyte-suffix"/> + <uri value="http://loinc.org/property/analyte-suffix"/> + <description value="The suffix part of the analyte, if present, e.g., Ab or DNA"/> + <type value="Coding"/> + </property> + <property> + <code value="analyte-numerator"/> + <uri value="http://loinc.org/property/analyte-numerator"/> + <description value="The numerator part of the analyte, i.e., everything before the slash in analytes that contain a divisor"/> + <type value="Coding"/> + </property> + <property> + <code value="analyte-divisor"/> + <uri value="http://loinc.org/property/analyte-divisor"/> + <description value="The divisor part of the analyte, if present, i.e., after the slash and before the first carat"/> + <type value="Coding"/> + </property> + <property> + <code value="analyte-divisor-suffix"/> + <uri value="http://loinc.org/property/analyte-divisor-suffix"/> + <description value="The suffix part of the divisor, if present"/> + <type value="Coding"/> + </property> + <property> + <code value="challenge"/> + <uri value="http://loinc.org/property/challenge"/> + <description value="Second sub-part of the Component, i.e., after the first carat"/> + <type value="Coding"/> + </property> + <property> + <code value="adjustment"/> + <uri value="http://loinc.org/property/adjustment"/> + <description value="Third sub-part of the Component, i.e., after the second carat"/> + <type value="Coding"/> + </property> + <property> + <code value="count"/> + <uri value="http://loinc.org/property/count"/> + <description value="Fourth sub-part of the Component, i.e., after the third carat"/> + <type value="Coding"/> + </property> + <property> + <code value="time-core"/> + <uri value="http://loinc.org/property/time-core"/> + <description value="The primary part of the Time"/> + <type value="Coding"/> + </property> + <property> + <code value="time-modifier"/> + <uri value="http://loinc.org/property/time-modifier"/> + <description value="The modifier of the Time value, such as mean or max"/> + <type value="Coding"/> + </property> + <property> + <code value="system-core"/> + <uri value="http://loinc.org/property/system-core"/> + <description value="The primary part of the System, i.e., without the super system"/> + <type value="Coding"/> + </property> + <property> + <code value="super-system"/> + <uri value="http://loinc.org/property/super-system"/> + <description value="The super system part of the System, if present. The super system represents the source of the specimen when the source is someone or something other than the patient whose chart the result will be stored in. For example, fetus is the super system for measurements done on obstetric ultrasounds, because the fetus is being measured and that measurement is being recorded in the patient's (mother's) chart."/> + <type value="Coding"/> + </property> + <property> + <code value="analyte-gene"/> + <uri value="http://loinc.org/property/analyte-gene"/> + <description value="The specific gene represented in the analyte"/> + <type value="Coding"/> + </property> + <property> + <code value="category"/> + <uri value="http://loinc.org/property/category"/> + <description value="A single LOINC term can be assigned one or more categories based on both programmatic and manual tagging. Category properties also utilize LOINC Class Parts."/> + <type value="Coding"/> + </property> + <property> + <code value="search"/> + <uri value="http://loinc.org/property/search"/> + <description value="Synonyms, fragments, and other Parts that are linked to a term to enable more encompassing search results."/> + <type value="Coding"/> + </property> + + <!-- + LOINC/RSNA Radiology Playbook properties. These apply only to terms in the LOINC/RSNA Radiology Playbook File. + Notes: + Properties are specified as type "coding", which are represented by LOINC Part codes (LP-) and their display names. + The attribute names here use FHIR styled names rather than their original LOINC style names because the original names contain periods. + --> + + <property> + <code value="rad-modality-modality-type"/> + <uri value="http://loinc.org/property/rad-modality-modality-type"/> + <description value="Modality is used to represent the device used to acquire imaging information."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-modality-modality-subtype"/> + <uri value="http://loinc.org/property/rad-modality-modality-subtype"/> + <description value="Modality subtype may be optionally included to signify a particularly common or evocative configuration of the modality."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-anatomic-location-region-imaged"/> + <uri value="http://loinc.org/property/rad-anatomic-location-region-imaged"/> + <description value="The Anatomic Location Region Imaged attribute is used in two ways: as a coarse-grained descriptor of the area imaged and a grouper for finding related imaging exams; or, it is used just as a grouper."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-anatomic-location-imaging-focus"/> + <uri value="http://loinc.org/property/rad-anatomic-location-imaging-focus"/> + <description value="The Anatomic Location Imaging Focus is a more fine-grained descriptor of the specific target structure of an imaging exam. In many areas, the focus should be a specific organ."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-anatomic-location-laterality-presence"/> + <uri value="http://loinc.org/property/rad-anatomic-location-laterality-presence"/> + <description value="Radiology Exams that require laterality to be specified in order to be performed are signified with an Anatomic Location Laterality Presence attribute set to 'True'"/> + <type value="Coding"/> + </property> + <property> + <code value="rad-anatomic-location-laterality"/> + <uri value="http://loinc.org/property/rad-anatomic-location-laterality"/> + <description value="Radiology exam Laterality is specified as one of: Left, Right, Bilateral, Unilateral, Unspecified"/> + <type value="Coding"/> + </property> + <property> + <code value="rad-view-aggregation"/> + <uri value="http://loinc.org/property/rad-view-aggregation"/> + <description value="Aggregation describes the extent of the imaging performed, whether in quantitative terms (e.g., '3 or more views') or subjective terms (e.g., 'complete')."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-view-view-type"/> + <uri value="http://loinc.org/property/rad-view-view-type"/> + <description value="View type names specific views, such as 'lateral' or 'AP'."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-maneuver-maneuver-type"/> + <uri value="http://loinc.org/property/rad-maneuver-maneuver-type"/> + <description value="Maneuver type indicates an action taken with the goal of elucidating or testing a dynamic aspect of the anatomy."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-timing"/> + <uri value="http://loinc.org/property/rad-timing"/> + <description value="The Timing/Existence property used in conjunction with pharmaceutical and maneuver properties. It specifies whether or not the imaging occurs in the presence of the administered pharmaceutical or a maneuver designed to test some dynamic aspect of anatomy or physiology ."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-pharmaceutical-substance-given"/> + <uri value="http://loinc.org/property/rad-pharmaceutical-substance-given"/> + <description value="The Pharmaceutical Substance Given specifies administered contrast agents, radiopharmaceuticals, medications, or other clinically important agents and challenges during the imaging procedure."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-pharmaceutical-route"/> + <uri value="http://loinc.org/property/rad-pharmaceutical-route"/> + <description value="Route specifies the route of administration of the pharmaceutical."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-reason-for-exam"/> + <uri value="http://loinc.org/property/rad-reason-for-exam"/> + <description value="Reason for exam is used to describe a clinical indication or a purpose for the study."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-guidance-for-presence"/> + <uri value="http://loinc.org/property/rad-guidance-for-presence"/> + <description value="Guidance for.Presence indicates when a procedure is guided by imaging."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-guidance-for-approach"/> + <uri value="http://loinc.org/property/rad-guidance-for-approach"/> + <description value="Guidance for.Approach refers to the primary route of access used, such as percutaneous, transcatheter, or transhepatic."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-guidance-for-action"/> + <uri value="http://loinc.org/property/rad-guidance-for-action"/> + <description value="Guidance for.Action indicates the intervention performed, such as biopsy, aspiration, or ablation."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-guidance-for-object"/> + <uri value="http://loinc.org/property/rad-guidance-for-object"/> + <description value="Guidance for.Object specifies the target of the action, such as mass, abscess or cyst."/> + <type value="Coding"/> + </property> + <property> + <code value="rad-subject"/> + <uri value="http://loinc.org/property/rad-subject"/> + <description value="Subject is intended for use when there is a need to distinguish between the patient associated with an imaging study, and the target of the study."/> + <type value="Coding"/> + </property> + + <!-- + Document Ontology properties. + These apply only to terms in the LOINC Document Ontology File + Notes + Properties are specified as type "coding", which are represented by LOINC Part codes (LP-) and their display names. + The attribute names here use FHIR styled names rather than their original LOINC style names because those contain periods. + --> + + <property> + <code value="document-kind"/> + <uri value="http://loinc.org/property/document-kind"/> + <description value="Characterizes the general structure of the document at a macro level."/> + <type value="Coding"/> + </property> + <property> + <code value="document-role"/> + <uri value="http://loinc.org/property/document-role"/> + <description value="Characterizes the training or professional level of the author of the document, but does not break down to specialty or subspecialty."/> + <type value="Coding"/> + </property> + <property> + <code value="document-setting"/> + <uri value="http://loinc.org/property/document-setting"/> + <description value="Setting is a modest extension of CMS’s coarse definition of care settings, such as outpatient, hospital, etc. Setting is not equivalent to location, which typically has more locally defined meanings."/> + <type value="Coding"/> + </property> + <property> + <code value="document-subject-matter-domain"/> + <uri value="http://loinc.org/property/document-subject-matter-domain"/> + <description value="Characterizes the clinical domain that is the subject of the document. For example, Internal Medicine, Neurology, Physical Therapy, etc."/> + <type value="Coding"/> + </property> + <property> + <code value="document-type-of-service"/> + <uri value="http://loinc.org/property/document-type-of-service"/> + <description value="Characterizes the kind of service or activity provided to/for the patient (or other subject of the service) that is described in the document."/> + <type value="Coding"/> + </property> + + <!-- Answer list related properties --> + <property> + <code value="answers-for"/> + <uri value="http://loinc.org/property/answers-for"/> + <description value="A LOINC Code for which this answer list is used."/> + <type value="Coding"/> + </property> + + <!-- Note for future consideration. These are properties of LA codes in the context of a particular list. Not global properties. + <property> + <code value="sequence"/> + <uri value="http://loinc.org/property/sequence"/> + <description value="Sequence Number of a answer in a set of answers (LA- codes only)"/> + <type value="integer"/> + </property> + <property> + <code value="score"/> + <uri value="http://loinc.org/property/score"/> + <description value="Score assigned to an answer (LA- codes only)"/> + <type value="integer"/> + </property> + --> </CodeSystem> + diff --git a/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loincupload.properties b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loincupload.properties new file mode 100644 index 00000000000..986d2afdaa0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/resources/ca/uhn/fhir/jpa/term/loinc/loincupload.properties @@ -0,0 +1,83 @@ +################# +### MANDATORY ### +################# + +# Answer lists (ValueSets of potential answers/values for LOINC "questions") +## File must be present +loinc.answerlist.file=AccessoryFiles/AnswerFile/AnswerList.csv +# Answer list links (connects LOINC observation codes to answer list codes) +## File must be present +loinc.answerlist.link.file=AccessoryFiles/AnswerFile/LoincAnswerListLink.csv + +# Document ontology +## File must be present +loinc.document.ontology.file=AccessoryFiles/DocumentOntology/DocumentOntology.csv + +# LOINC codes +## File must be present +loinc.file=LoincTable/Loinc.csv + +# LOINC hierarchy +## File must be present +loinc.hierarchy.file=AccessoryFiles/MultiAxialHierarchy/MultiAxialHierarchy.csv + +# IEEE medical device codes +## File must be present +loinc.ieee.medical.device.code.mapping.table.file=AccessoryFiles/LoincIeeeMedicalDeviceCodeMappingTable/LoincIeeeMedicalDeviceCodeMappingTable.csv + +# Imaging document codes +## File must be present +loinc.imaging.document.codes.file=AccessoryFiles/ImagingDocuments/ImagingDocumentCodes.csv + +# Part +## File must be present +loinc.part.file=AccessoryFiles/PartFile/Part.csv + +# Part link +## File must be present +loinc.part.link.primary.file=AccessoryFiles/PartFile/LoincPartLink_Primary.csv +loinc.part.link.supplementary.file=AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv + +# Part related code mapping +## File must be present +loinc.part.related.code.mapping.file=AccessoryFiles/PartFile/PartRelatedCodeMapping.csv + +# RSNA playbook +## File must be present +loinc.rsna.playbook.file=AccessoryFiles/LoincRsnaRadiologyPlaybook/LoincRsnaRadiologyPlaybook.csv + +# Top 2000 codes - SI +## File must be present +loinc.top2000.common.lab.results.si.file=AccessoryFiles/Top2000Results/SI/Top2000CommonLabResultsSi.csv +# Top 2000 codes - US +## File must be present +loinc.top2000.common.lab.results.us.file=AccessoryFiles/Top2000Results/US/Top2000CommonLabResultsUs.csv + +# Universal lab order ValueSet +## File must be present +loinc.universal.lab.order.valueset.file=AccessoryFiles/LoincUniversalLabOrdersValueSet/LoincUniversalLabOrdersValueSet.csv + +################ +### OPTIONAL ### +################ + +# This is the version identifier for the answer list file +## Key may be omitted +loinc.answerlist.version=Beta.1 + +# This is the version identifier for uploaded ConceptMap resources +## Key may be omitted +loinc.conceptmap.version=Beta.1 + +# Group +## Default value if key not provided: AccessoryFiles/GroupFile/Group.csv +## File may be omitted +loinc.group.file=AccessoryFiles/GroupFile/Group.csv +# Group terms +## Default value if key not provided: AccessoryFiles/GroupFile/GroupLoincTerms.csv +## File may be omitted +loinc.group.terms.file=AccessoryFiles/GroupFile/GroupLoincTerms.csv +# Parent group +## Default value if key not provided: AccessoryFiles/GroupFile/ParentGroup.csv +## File may be omitted +loinc.parent.group.file=AccessoryFiles/GroupFile/ParentGroup.csv diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/TerminologyUploaderProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/TerminologyUploaderProviderR4Test.java index cfb73e25a0b..e53eeb26014 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/TerminologyUploaderProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/TerminologyUploaderProviderR4Test.java @@ -492,7 +492,8 @@ public class TerminologyUploaderProviderR4Test extends BaseResourceProviderR4Tes addFile(zos, LOINC_GROUP_FILE_DEFAULT.getCode()); addFile(zos, LOINC_GROUP_TERMS_FILE_DEFAULT.getCode()); addFile(zos, LOINC_PARENT_GROUP_FILE_DEFAULT.getCode()); - addFile(zos, LOINC_PART_LINK_FILE_DEFAULT.getCode()); + addFile(zos, LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()); + addFile(zos, LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()); addFile(zos, LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()); addFile(zos, LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode()); addFile(zos, LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationDstu3Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationDstu3Test.java index dd4abe50c7f..6a06243fde9 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationDstu3Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationDstu3Test.java @@ -86,7 +86,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test { Set<String> codes = toExpandedCodes(expanded); ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded)); ourLog.info("Codes: {}", codes); - assertThat(codes, containsInAnyOrder("10019-8", "10013-1", "10014-9", "10016-4", "17788-1", "10000-8", "10017-2", "10015-6", "10020-6", "10018-0")); + assertThat(codes, containsInAnyOrder("10013-1")); // Search by display name input = new ValueSet(); @@ -101,7 +101,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test { expanded = myValueSetDao.expand(input, null); codes = toExpandedCodes(expanded); ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded)); - assertThat(codes, containsInAnyOrder("10019-8", "10013-1", "10014-9", "10016-4", "17788-1", "10000-8", "10017-2", "10015-6", "10020-6", "10018-0")); + assertThat(codes, containsInAnyOrder("10013-1")); // Search by something that doesn't match input = new ValueSet(); @@ -149,7 +149,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test { Set<String> codes = toExpandedCodes(expanded); ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded)); ourLog.info("Codes: {}", codes); - assertThat(codes, containsInAnyOrder("10019-8", "10013-1", "10014-9", "10000-8", "10016-4", "10017-2", "10015-6", "10020-6", "10018-0")); + assertThat(codes, containsInAnyOrder("10013-1")); } @Test @@ -188,7 +188,7 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test { TerminologyLoaderSvcLoincTest.addLoincMandatoryFilesToZip(files); myLoader.loadLoinc(files.getFiles(), mySrd); - IValidationSupport.LookupCodeResult result = myCodeSystemDao.lookupCode(new StringType("17788-1"), new StringType(ITermLoaderSvc.LOINC_URI), null, mySrd); + IValidationSupport.LookupCodeResult result = myCodeSystemDao.lookupCode(new StringType("10013-1"), new StringType(ITermLoaderSvc.LOINC_URI), null, mySrd); Parameters parameters = (Parameters) result.toParameters(myFhirCtx, null); ourLog.info(myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(parameters)); @@ -196,8 +196,8 @@ public class TerminologyLoaderSvcIntegrationDstu3Test extends BaseJpaDstu3Test { Optional<Coding> propertyValue = findProperty(parameters, "COMPONENT"); assertTrue(propertyValue.isPresent()); assertEquals(ITermLoaderSvc.LOINC_URI, propertyValue.get().getSystem()); - assertEquals("LP19258-0", propertyValue.get().getCode()); - assertEquals("Large unstained cells/100 leukocytes", propertyValue.get().getDisplay()); + assertEquals("LP31101-6", propertyValue.get().getCode()); + assertEquals("R' wave amplitude.lead I", propertyValue.get().getDisplay()); } @Test diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java index 1cef209f864..342c82421eb 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcLoincTest.java @@ -76,18 +76,20 @@ public class TerminologyLoaderSvcLoincTest extends BaseLoaderTest { // Normal LOINC code code = concepts.get("10013-1"); assertEquals("10013-1", code.getCode()); + // Coding Property assertEquals(ITermLoaderSvc.LOINC_URI, code.getCodingProperties("PROPERTY").get(0).getSystem()); assertEquals("LP6802-5", code.getCodingProperties("PROPERTY").get(0).getCode()); assertEquals("Elpot", code.getCodingProperties("PROPERTY").get(0).getDisplay()); - assertEquals("EKG.MEAS", code.getStringProperty("CLASS")); + // String Property + assertEquals("2", code.getStringProperty("CLASSTYPE")); assertEquals("R' wave amplitude in lead I", code.getDisplay()); - + // Coding Property from Part File + assertEquals(ITermLoaderSvc.LOINC_URI, code.getCodingProperties("TIME_ASPCT").get(0).getSystem()); + assertEquals("LP6960-1", code.getCodingProperties("TIME_ASPCT").get(0).getCode()); + assertEquals("Pt", code.getCodingProperties("TIME_ASPCT").get(0).getDisplay()); // Code with component that has a divisor code = concepts.get("17788-1"); assertEquals("17788-1", code.getCode()); - assertEquals(1, code.getCodingProperties("COMPONENT").size()); - assertEquals("http://loinc.org", code.getCodingProperties("COMPONENT").get(0).getSystem()); - assertEquals("LP19258-0", code.getCodingProperties("COMPONENT").get(0).getCode()); // LOINC code with answer code = concepts.get("61438-8"); @@ -379,7 +381,8 @@ public class TerminologyLoaderSvcLoincTest extends BaseLoaderTest { theFiles.addFileZip("/loinc/", LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_ANSWERLIST_LINK_DUPLICATE_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_PART_FILE_DEFAULT.getCode()); - theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_DEFAULT.getCode()); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()); + theFiles.addFileZip("/loinc/", LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode()); theFiles.addFileZip("/loinc/", LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()); @@ -413,7 +416,10 @@ public class TerminologyLoaderSvcLoincTest extends BaseLoaderTest { assertEquals(ITermLoaderSvc.LOINC_URI, code.getCodingProperties("PROPERTY").get(0).getSystem()); assertEquals("LP6802-5", code.getCodingProperties("PROPERTY").get(0).getCode()); assertEquals("Elpot", code.getCodingProperties("PROPERTY").get(0).getDisplay()); - assertEquals("EKG.MEAS", code.getStringProperty("CLASS")); + assertEquals(ITermLoaderSvc.LOINC_URI, code.getCodingProperties("PROPERTY").get(0).getSystem()); + assertEquals("LP6802-5", code.getCodingProperties("PROPERTY").get(0).getCode()); + assertEquals("Elpot", code.getCodingProperties("PROPERTY").get(0).getDisplay()); + assertEquals("2", code.getStringProperty("CLASSTYPE")); assertEquals("R' wave amplitude in lead I", code.getDisplay()); // Codes with parent and child properties diff --git a/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv deleted file mode 100644 index 07277640bd7..00000000000 --- a/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink.csv +++ /dev/null @@ -1,10 +0,0 @@ -"LoincNumber","LongCommonName","PartNumber","PartName","PartTypeName","LinkTypeName" -"10000-8","R wave duration in lead AVR","LP31088-5","R wave duration.lead AVR","COMPONENT","Primary" -"10000-8","R wave duration in lead AVR","LP6244-0","EKG","METHOD","Primary" -"10000-8","R wave duration in lead AVR","LP6879-3","Time","PROPERTY","Primary" -"10000-8","R wave duration in lead AVR","LP6960-1","Pt","TIME","Primary" -"10000-8","R wave duration in lead AVR","LP7289-4","Heart","SYSTEM","Primary" -"10000-8","R wave duration in lead AVR","LP7753-9","Qn","SCALE","Primary" -"10000-8","R wave duration in lead AVR","LP7795-0","EKG.MEAS","CLASS","Primary" -"10000-8","R wave duration in lead AVR","LP14259-3","Lead","COMPONENT","Search" -"10000-8","R wave duration in lead AVR","LP14744-4","Duration","COMPONENT","Search" diff --git a/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Primary.csv b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Primary.csv new file mode 100644 index 00000000000..8dbe575ffd2 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Primary.csv @@ -0,0 +1,7 @@ +"LoincNumber","LongCommonName" ,"PartNumber","PartName" ,"PartCodeSystem" ,"PartTypeName","LinkTypeName","Property" +"10013-1" ,"R' wave amplitude in lead I","LP31101-6" ,"R' wave amplitude.lead I","http://loinc.org","COMPONENT" ,"Primary" ,"http://loinc.org/property/COMPONENT" +"10013-1" ,"R' wave amplitude in lead I","LP6802-5" ,"Elpot" ,"http://loinc.org","PROPERTY" ,"Primary" ,"http://loinc.org/property/PROPERTY" +"10013-1" ,"R' wave amplitude in lead I","LP6960-1" ,"Pt" ,"http://loinc.org","TIME" ,"Primary" ,"http://loinc.org/property/TIME_ASPCT" +"10013-1" ,"R' wave amplitude in lead I","LP7289-4" ,"Heart" ,"http://loinc.org","SYSTEM" ,"Primary" ,"http://loinc.org/property/SYSTEM" +"10013-1" ,"R' wave amplitude in lead I","LP7753-9" ,"Qn" ,"http://loinc.org","SCALE" ,"Primary" ,"http://loinc.org/property/SCALE_TYP" +"10013-1" ,"R' wave amplitude in lead I","LP6244-0" ,"EKG" ,"http://loinc.org","METHOD" ,"Primary" ,"http://loinc.org/property/METHOD_TYP" diff --git a/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv new file mode 100644 index 00000000000..869c2c1651d --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/resources/loinc/AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv @@ -0,0 +1,13 @@ +"LoincNumber","LongCommonName" ,"PartNumber","PartName" ,"PartCodeSystem" ,"PartTypeName","LinkTypeName" ,"Property" +"10013-1" ,"R' wave amplitude in lead I","LP31101-6" ,"R' wave amplitude.lead I","http://loinc.org","COMPONENT" ,"DetailedModel" ,"http://loinc.org/property/analyte" +"10013-1" ,"R' wave amplitude in lead I","LP6802-5" ,"Elpot" ,"http://loinc.org","PROPERTY" ,"DetailedModel" ,"http://loinc.org/property/PROPERTY" +"10013-1" ,"R' wave amplitude in lead I","LP6960-1" ,"Pt" ,"http://loinc.org","TIME" ,"DetailedModel" ,"http://loinc.org/property/time-core" +"10013-1" ,"R' wave amplitude in lead I","LP7289-4" ,"Heart" ,"http://loinc.org","SYSTEM" ,"DetailedModel" ,"http://loinc.org/property/system-core" +"10013-1" ,"R' wave amplitude in lead I","LP7753-9" ,"Qn" ,"http://loinc.org","SCALE" ,"DetailedModel" ,"http://loinc.org/property/SCALE_TYP" +"10013-1" ,"R' wave amplitude in lead I","LP6244-0" ,"EKG" ,"http://loinc.org","METHOD" ,"DetailedModel" ,"http://loinc.org/property/METHOD_TYP" +"10013-1" ,"R' wave amplitude in lead I","LP31101-6" ,"R' wave amplitude.lead I","http://loinc.org","COMPONENT" ,"SyntaxEnhancement","http://loinc.org/property/analyte-core" +"10013-1" ,"R' wave amplitude in lead I","LP190563-9","Cardiology" ,"http://loinc.org","CLASS" ,"Metadata" ,"http://loinc.org/property/category" +"10013-1" ,"R' wave amplitude in lead I","LP29708-2" ,"Cardiology" ,"http://loinc.org","CLASS" ,"Metadata" ,"http://loinc.org/property/category" +"10013-1" ,"R' wave amplitude in lead I","LP7787-7" ,"Clinical" ,"http://loinc.org","CLASS" ,"Metadata" ,"http://loinc.org/property/category" +"10013-1" ,"R' wave amplitude in lead I","LP7795-0" ,"EKG measurements" ,"http://loinc.org","CLASS" ,"Metadata" ,"http://loinc.org/property/category" +"10013-1" ,"R' wave amplitude in lead I","LP7795-0" ,"EKG.MEAS" ,"http://loinc.org","CLASS" ,"Metadata" ,"http://loinc.org/property/CLASS" diff --git a/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload.properties b/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload.properties index 3c01c55a2a9..986d2afdaa0 100644 --- a/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload.properties +++ b/hapi-fhir-jpaserver-base/src/test/resources/loinc/loincupload.properties @@ -3,68 +3,57 @@ ################# # Answer lists (ValueSets of potential answers/values for LOINC "questions") -## Default value if key not provided: AccessoryFiles/AnswerFile/AnswerList.csv ## File must be present loinc.answerlist.file=AccessoryFiles/AnswerFile/AnswerList.csv # Answer list links (connects LOINC observation codes to answer list codes) -## Default value if key not provided: AccessoryFiles/AnswerFile/LoincAnswerListLink.csv ## File must be present loinc.answerlist.link.file=AccessoryFiles/AnswerFile/LoincAnswerListLink.csv # Document ontology -## Default value if key not provided: AccessoryFiles/DocumentOntology/DocumentOntology.csv ## File must be present loinc.document.ontology.file=AccessoryFiles/DocumentOntology/DocumentOntology.csv # LOINC codes -## Default value if key not provided: LoincTable/Loinc.csv ## File must be present loinc.file=LoincTable/Loinc.csv # LOINC hierarchy -## Default value if key not provided: AccessoryFiles/MultiAxialHierarchy/MultiAxialHierarchy.csv ## File must be present loinc.hierarchy.file=AccessoryFiles/MultiAxialHierarchy/MultiAxialHierarchy.csv # IEEE medical device codes -## Default value if key not provided: AccessoryFiles/LoincIeeeMedicalDeviceCodeMappingTable/LoincIeeeMedicalDeviceCodeMappingTable.csv ## File must be present loinc.ieee.medical.device.code.mapping.table.file=AccessoryFiles/LoincIeeeMedicalDeviceCodeMappingTable/LoincIeeeMedicalDeviceCodeMappingTable.csv # Imaging document codes -## Default value if key not provided: AccessoryFiles/ImagingDocuments/ImagingDocumentCodes.csv ## File must be present loinc.imaging.document.codes.file=AccessoryFiles/ImagingDocuments/ImagingDocumentCodes.csv # Part -## Default value if key not provided: AccessoryFiles/PartFile/Part.csv ## File must be present loinc.part.file=AccessoryFiles/PartFile/Part.csv + # Part link -## Default value if key not provided: AccessoryFiles/PartFile/LoincPartLink.csv ## File must be present -loinc.part.link.file=AccessoryFiles/PartFile/LoincPartLink.csv +loinc.part.link.primary.file=AccessoryFiles/PartFile/LoincPartLink_Primary.csv +loinc.part.link.supplementary.file=AccessoryFiles/PartFile/LoincPartLink_Supplementary.csv + # Part related code mapping -## Default value if key not provided: AccessoryFiles/PartFile/PartRelatedCodeMapping.csv ## File must be present loinc.part.related.code.mapping.file=AccessoryFiles/PartFile/PartRelatedCodeMapping.csv # RSNA playbook -## Default value if key not provided: AccessoryFiles/LoincRsnaRadiologyPlaybook/LoincRsnaRadiologyPlaybook.csv ## File must be present loinc.rsna.playbook.file=AccessoryFiles/LoincRsnaRadiologyPlaybook/LoincRsnaRadiologyPlaybook.csv # Top 2000 codes - SI -## Default value if key not provided: AccessoryFiles/Top2000Results/SI/Top2000CommonLabResultsSi.csv ## File must be present loinc.top2000.common.lab.results.si.file=AccessoryFiles/Top2000Results/SI/Top2000CommonLabResultsSi.csv # Top 2000 codes - US -## Default value if key not provided: AccessoryFiles/Top2000Results/US/Top2000CommonLabResultsUs.csv ## File must be present loinc.top2000.common.lab.results.us.file=AccessoryFiles/Top2000Results/US/Top2000CommonLabResultsUs.csv # Universal lab order ValueSet -## Default value if key not provided: AccessoryFiles/LoincUniversalLabOrdersValueSet/LoincUniversalLabOrdersValueSet.csv ## File must be present loinc.universal.lab.order.valueset.file=AccessoryFiles/LoincUniversalLabOrdersValueSet/LoincUniversalLabOrdersValueSet.csv diff --git a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java index c06e09405c0..b695d947ed7 100644 --- a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java +++ b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/FhirTerserR4Test.java @@ -6,10 +6,30 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.model.api.annotation.Block; import ca.uhn.fhir.parser.DataFormatException; import org.hamcrest.Matchers; -import org.hl7.fhir.instance.model.api.*; -import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.instance.model.api.IBase; +import org.hl7.fhir.instance.model.api.IBaseExtension; +import org.hl7.fhir.instance.model.api.IBaseReference; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Enumeration; +import org.hl7.fhir.r4.model.Enumerations; +import org.hl7.fhir.r4.model.Extension; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.MarkdownType; +import org.hl7.fhir.r4.model.Money; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Organization; +import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Patient.LinkType; +import org.hl7.fhir.r4.model.Practitioner; +import org.hl7.fhir.r4.model.PrimitiveType; +import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.SimpleQuantity; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.ValueSet; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Test; @@ -19,11 +39,22 @@ import org.slf4j.LoggerFactory; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; From 5bf1e94a236e7ef2ec5380690ac8405897095101 Mon Sep 17 00:00:00 2001 From: ianmarshall <ian@simpatico.ai> Date: Fri, 19 Jun 2020 17:26:45 -0400 Subject: [PATCH 10/22] Fix to accomodate MySQL and MariaDB. --- .../java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java index 6d4eaf94bd7..b17b9951e21 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/TermValueSetConceptView.java @@ -38,7 +38,7 @@ import java.io.Serializable; * Note about the CONCAT function below- We need a primary key (an @Id) column * because hibernate won't allow the view the function without it, but */ - "SELECT CONCAT(vsc.PID, CONCAT(' ', vscd.PID)) AS PID, " + + "SELECT CONCAT_WS(' ', vsc.PID, vscd.PID) AS PID, " + " vsc.PID AS CONCEPT_PID, " + " vsc.VALUESET_PID AS CONCEPT_VALUESET_PID, " + " vsc.VALUESET_ORDER AS CONCEPT_VALUESET_ORDER, " + From 314df430913f1e2d269a482aabc0701ee8a3be6f Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk <diederik.muylwyk@gmail.com> Date: Mon, 22 Jun 2020 16:52:52 -0400 Subject: [PATCH 11/22] BaseValidationSupportWrapper.expandValueSet(...) and ValidationSupportChain.expandValueSet(...) were incorrectly replacing expansion options (i.e. offset and count) with null. --- .../r5/ResourceProviderR5ValueSetTest.java | 362 +++++++++++++++++- .../support/BaseValidationSupportWrapper.java | 2 +- .../support/ValidationSupportChain.java | 2 +- 3 files changed, 363 insertions(+), 3 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java index b7dd83f9dc9..905713fe258 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java @@ -301,6 +301,122 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { } + @Test + public void testExpandByIdWithPreExpansionWithOffset() throws Exception { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + Parameters respParam = ourClient + .operation() + .onInstance(myExtensionalVsId) + .named("expand") + .withParameter(Parameters.class, "offset", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"1\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1000\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"11378-7\"/>\n", + "<display value=\"Systolic blood pressure at First encounter\"/>\n", + "</contains>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"8493-9\"/>\n", + "<display value=\"Systolic blood pressure 10 hour minimum\"/>\n", + "</contains>")); + + } + + @Test + public void testExpandByIdWithPreExpansionWithCount() throws Exception { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + Parameters respParam = ourClient + .operation() + .onInstance(myExtensionalVsId) + .named("expand") + .withParameter(Parameters.class, "count", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"0\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"0\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"8450-9\"/>\n", + "<display value=\"Systolic blood pressure--expiration\"/>\n", + "</contains>\n", + "</expansion>")); + + } + + @Test + public void testExpandByIdWithPreExpansionWithOffsetAndCount() throws Exception { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + Parameters respParam = ourClient + .operation() + .onInstance(myExtensionalVsId) + .named("expand") + .withParameter(Parameters.class, "offset", new IntegerType(1)) + .andParameter("count", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"1\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"11378-7\"/>\n", + "<display value=\"Systolic blood pressure at First encounter\"/>\n", + "</contains>\n", + "</expansion>")); + + } + @Test public void testExpandByIdWithFilter() throws Exception { loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); @@ -402,6 +518,125 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { } + @Test + public void testExpandByUrlWithPreExpansionWithOffset() throws Exception { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + Parameters respParam = ourClient + .operation() + .onType(ValueSet.class) + .named("expand") + .withParameter(Parameters.class, "url", new UriType("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2")) + .andParameter("offset", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"1\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1000\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"11378-7\"/>\n", + "<display value=\"Systolic blood pressure at First encounter\"/>\n", + "</contains>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"8493-9\"/>\n", + "<display value=\"Systolic blood pressure 10 hour minimum\"/>\n", + "</contains>")); + + } + + @Test + public void testExpandByUrlWithPreExpansionWithCount() throws Exception { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + Parameters respParam = ourClient + .operation() + .onType(ValueSet.class) + .named("expand") + .withParameter(Parameters.class, "url", new UriType("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2")) + .andParameter("count", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"0\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"0\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"8450-9\"/>\n", + "<display value=\"Systolic blood pressure--expiration\"/>\n", + "</contains>\n", + "</expansion>")); + + } + + @Test + public void testExpandByUrlWithPreExpansionWithOffsetAndCount() throws Exception { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + Parameters respParam = ourClient + .operation() + .onType(ValueSet.class) + .named("expand") + .withParameter(Parameters.class, "url", new UriType("http://www.healthintersections.com.au/fhir/ValueSet/extensional-case-2")) + .andParameter("offset", new IntegerType(1)) + .andParameter("count", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"1\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"11378-7\"/>\n", + "<display value=\"Systolic blood pressure at First encounter\"/>\n", + "</contains>\n", + "</expansion>")); + + } + @Test public void testExpandByUrlWithPreExpansionAndBogusUrl() throws Exception { myDaoConfig.setPreExpandValueSets(true); @@ -448,7 +683,7 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { public void testExpandByValueSetWithPreExpansion() throws IOException { myDaoConfig.setPreExpandValueSets(true); - loadAndPersistCodeSystem(HTTPVerb.POST); + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); ValueSet toExpand = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml"); @@ -469,6 +704,131 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { } + @Test + public void testExpandByValueSetWithPreExpansionWithOffset() throws IOException { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + ValueSet toExpand = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml"); + + Parameters respParam = ourClient + .operation() + .onType(ValueSet.class) + .named("expand") + .withParameter(Parameters.class, "valueSet", toExpand) + .andParameter("offset", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"1\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1000\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"11378-7\"/>\n", + "<display value=\"Systolic blood pressure at First encounter\"/>\n", + "</contains>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"8493-9\"/>\n", + "<display value=\"Systolic blood pressure 10 hour minimum\"/>\n", + "</contains>")); + + } + + @Test + public void testExpandByValueSetWithPreExpansionWithCount() throws IOException { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + ValueSet toExpand = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml"); + + Parameters respParam = ourClient + .operation() + .onType(ValueSet.class) + .named("expand") + .withParameter(Parameters.class, "valueSet", toExpand) + .andParameter("count", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"0\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"0\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"8450-9\"/>\n", + "<display value=\"Systolic blood pressure--expiration\"/>\n", + "</contains>\n", + "</expansion>")); + + } + + @Test + public void testExpandByValueSetWithPreExpansionWithOffsetAndCount() throws IOException { + myDaoConfig.setPreExpandValueSets(true); + + loadAndPersistCodeSystemAndValueSet(HTTPVerb.POST); + myTermSvc.preExpandDeferredValueSetsToTerminologyTables(); + + ValueSet toExpand = loadResourceFromClasspath(ValueSet.class, "/extensional-case-3-vs.xml"); + + Parameters respParam = ourClient + .operation() + .onType(ValueSet.class) + .named("expand") + .withParameter(Parameters.class, "valueSet", toExpand) + .andParameter("offset", new IntegerType(1)) + .andParameter("count", new IntegerType(1)) + .execute(); + ValueSet expanded = (ValueSet) respParam.getParameter().get(0).getResource(); + + String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); + ourLog.info(resp); + assertThat(resp, stringContainsInOrder( + "<total value=\"24\"/>\n", + "<offset value=\"1\"/>\n", + "<parameter>\n", + "<name value=\"offset\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<parameter>\n", + "<name value=\"count\"/>\n", + "<valueInteger value=\"1\"/>\n", + "</parameter>\n", + "<contains>\n", + "<system value=\"http://acme.org\"/>\n", + "<code value=\"11378-7\"/>\n", + "<display value=\"Systolic blood pressure at First encounter\"/>\n", + "</contains>\n", + "</expansion>")); + + } + @Test public void testExpandInlineVsAgainstBuiltInCs() { createLocalVsPointingAtBuiltInCodeSystem(); diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/BaseValidationSupportWrapper.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/BaseValidationSupportWrapper.java index e18e1918f14..cc04a447d10 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/BaseValidationSupportWrapper.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/BaseValidationSupportWrapper.java @@ -70,7 +70,7 @@ public class BaseValidationSupportWrapper extends BaseValidationSupport { @Override public IValidationSupport.ValueSetExpansionOutcome expandValueSet(ValidationSupportContext theValidationSupportContext, ValueSetExpansionOptions theExpansionOptions, IBaseResource theValueSetToExpand) { - return myWrap.expandValueSet(theValidationSupportContext, null, theValueSetToExpand); + return myWrap.expandValueSet(theValidationSupportContext, theExpansionOptions, theValueSetToExpand); } @Override diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java index 3076d31826d..20a667e0b96 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/ValidationSupportChain.java @@ -134,7 +134,7 @@ public class ValidationSupportChain implements IValidationSupport { public ValueSetExpansionOutcome expandValueSet(ValidationSupportContext theValidationSupportContext, ValueSetExpansionOptions theExpansionOptions, IBaseResource theValueSetToExpand) { for (IValidationSupport next : myChain) { // TODO: test if code system is supported? - ValueSetExpansionOutcome expanded = next.expandValueSet(theValidationSupportContext, null, theValueSetToExpand); + ValueSetExpansionOutcome expanded = next.expandValueSet(theValidationSupportContext, theExpansionOptions, theValueSetToExpand); if (expanded != null) { return expanded; } From ced376f7ba0ecfa14003d58beb0914a474227b96 Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk <diederik.muylwyk@gmail.com> Date: Mon, 22 Jun 2020 17:02:08 -0400 Subject: [PATCH 12/22] Add changelog entry; fix typo. --- .../1895-fix-jpa-validation-performance-regression.yaml | 2 +- ...ix-valueset-expand-with-offset-and-count-parameters.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1933-fix-valueset-expand-with-offset-and-count-parameters.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1895-fix-jpa-validation-performance-regression.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1895-fix-jpa-validation-performance-regression.yaml index d7c082340b9..a6f7ebe888d 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1895-fix-jpa-validation-performance-regression.yaml +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1895-fix-jpa-validation-performance-regression.yaml @@ -1,5 +1,5 @@ --- type: fix issue: 1895 -title: "HAPI FHIR 5.0.0 introduced a regressin in JPA validator performance, where a number of unnecessary database lookups +title: "HAPI FHIR 5.0.0 introduced a regression in JPA validator performance, where a number of unnecessary database lookups were introduced. This has been corrected." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1933-fix-valueset-expand-with-offset-and-count-parameters.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1933-fix-valueset-expand-with-offset-and-count-parameters.yaml new file mode 100644 index 00000000000..4e9d7a3aaf0 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1933-fix-valueset-expand-with-offset-and-count-parameters.yaml @@ -0,0 +1,6 @@ +--- +type: fix +issue: 1933 +title: "`BaseValidationSupportWrapper.expandValueSet(...)` and `ValidationSupportChain.expandValueSet(...)` were + incorrectly replacing expansion options (i.e. offset and count) with `null`, causing these parameters to be + ignored when invoking the `ValueSet$expand` operation. This has been corrected." From ce0cb8916eda93b730e0bb787f60c9b977049a7f Mon Sep 17 00:00:00 2001 From: Diederik Muylwyk <diederik.muylwyk@gmail.com> Date: Mon, 22 Jun 2020 17:22:21 -0400 Subject: [PATCH 13/22] Improve tests. --- .../r5/ResourceProviderR5ValueSetTest.java | 273 +++++++----------- 1 file changed, 108 insertions(+), 165 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java index 905713fe258..58a6bf69e5d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5ValueSetTest.java @@ -318,27 +318,20 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"1\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1000\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"11378-7\"/>\n", - "<display value=\"Systolic blood pressure at First encounter\"/>\n", - "</contains>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"8493-9\"/>\n", - "<display value=\"Systolic blood pressure 10 hour minimum\"/>\n", - "</contains>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(1, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1000, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(23, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContains().get(0).getSystem()); + assertEquals("11378-7", expanded.getExpansion().getContains().get(0).getCode()); + assertEquals("Systolic blood pressure at First encounter", expanded.getExpansion().getContains().get(0).getDisplay()); + assertEquals("http://acme.org", expanded.getExpansion().getContains().get(1).getSystem()); + assertEquals("8493-9", expanded.getExpansion().getContains().get(1).getCode()); + assertEquals("Systolic blood pressure 10 hour minimum", expanded.getExpansion().getContains().get(1).getDisplay()); } @@ -359,23 +352,17 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"0\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"0\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"8450-9\"/>\n", - "<display value=\"Systolic blood pressure--expiration\"/>\n", - "</contains>\n", - "</expansion>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(0, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(0, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(1, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContainsFirstRep().getSystem()); + assertEquals("8450-9", expanded.getExpansion().getContainsFirstRep().getCode()); + assertEquals("Systolic blood pressure--expiration", expanded.getExpansion().getContainsFirstRep().getDisplay()); } @@ -397,23 +384,17 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"1\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"11378-7\"/>\n", - "<display value=\"Systolic blood pressure at First encounter\"/>\n", - "</contains>\n", - "</expansion>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(1, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(1, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContainsFirstRep().getSystem()); + assertEquals("11378-7", expanded.getExpansion().getContainsFirstRep().getCode()); + assertEquals("Systolic blood pressure at First encounter", expanded.getExpansion().getContainsFirstRep().getDisplay()); } @@ -536,27 +517,20 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"1\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1000\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"11378-7\"/>\n", - "<display value=\"Systolic blood pressure at First encounter\"/>\n", - "</contains>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"8493-9\"/>\n", - "<display value=\"Systolic blood pressure 10 hour minimum\"/>\n", - "</contains>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(1, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1000, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(23, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContains().get(0).getSystem()); + assertEquals("11378-7", expanded.getExpansion().getContains().get(0).getCode()); + assertEquals("Systolic blood pressure at First encounter", expanded.getExpansion().getContains().get(0).getDisplay()); + assertEquals("http://acme.org", expanded.getExpansion().getContains().get(1).getSystem()); + assertEquals("8493-9", expanded.getExpansion().getContains().get(1).getCode()); + assertEquals("Systolic blood pressure 10 hour minimum", expanded.getExpansion().getContains().get(1).getDisplay()); } @@ -578,23 +552,17 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"0\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"0\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"8450-9\"/>\n", - "<display value=\"Systolic blood pressure--expiration\"/>\n", - "</contains>\n", - "</expansion>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(0, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(0, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(1, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContainsFirstRep().getSystem()); + assertEquals("8450-9", expanded.getExpansion().getContainsFirstRep().getCode()); + assertEquals("Systolic blood pressure--expiration", expanded.getExpansion().getContainsFirstRep().getDisplay()); } @@ -617,23 +585,17 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"1\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"11378-7\"/>\n", - "<display value=\"Systolic blood pressure at First encounter\"/>\n", - "</contains>\n", - "</expansion>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(1, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(1, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContainsFirstRep().getSystem()); + assertEquals("11378-7", expanded.getExpansion().getContainsFirstRep().getCode()); + assertEquals("Systolic blood pressure at First encounter", expanded.getExpansion().getContainsFirstRep().getDisplay()); } @@ -724,27 +686,20 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"1\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1000\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"11378-7\"/>\n", - "<display value=\"Systolic blood pressure at First encounter\"/>\n", - "</contains>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"8493-9\"/>\n", - "<display value=\"Systolic blood pressure 10 hour minimum\"/>\n", - "</contains>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(1, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1000, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(23, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContains().get(0).getSystem()); + assertEquals("11378-7", expanded.getExpansion().getContains().get(0).getCode()); + assertEquals("Systolic blood pressure at First encounter", expanded.getExpansion().getContains().get(0).getDisplay()); + assertEquals("http://acme.org", expanded.getExpansion().getContains().get(1).getSystem()); + assertEquals("8493-9", expanded.getExpansion().getContains().get(1).getCode()); + assertEquals("Systolic blood pressure 10 hour minimum", expanded.getExpansion().getContains().get(1).getDisplay()); } @@ -768,23 +723,17 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"0\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"0\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"8450-9\"/>\n", - "<display value=\"Systolic blood pressure--expiration\"/>\n", - "</contains>\n", - "</expansion>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(0, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(0, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(1, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContainsFirstRep().getSystem()); + assertEquals("8450-9", expanded.getExpansion().getContainsFirstRep().getCode()); + assertEquals("Systolic blood pressure--expiration", expanded.getExpansion().getContainsFirstRep().getDisplay()); } @@ -809,23 +758,17 @@ public class ResourceProviderR5ValueSetTest extends BaseResourceProviderR5Test { String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(expanded); ourLog.info(resp); - assertThat(resp, stringContainsInOrder( - "<total value=\"24\"/>\n", - "<offset value=\"1\"/>\n", - "<parameter>\n", - "<name value=\"offset\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<parameter>\n", - "<name value=\"count\"/>\n", - "<valueInteger value=\"1\"/>\n", - "</parameter>\n", - "<contains>\n", - "<system value=\"http://acme.org\"/>\n", - "<code value=\"11378-7\"/>\n", - "<display value=\"Systolic blood pressure at First encounter\"/>\n", - "</contains>\n", - "</expansion>")); + + assertEquals(24, expanded.getExpansion().getTotal()); + assertEquals(1, expanded.getExpansion().getOffset()); + assertEquals("offset", expanded.getExpansion().getParameter().get(0).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(0).getValueIntegerType().getValue().intValue()); + assertEquals("count", expanded.getExpansion().getParameter().get(1).getName()); + assertEquals(1, expanded.getExpansion().getParameter().get(1).getValueIntegerType().getValue().intValue()); + assertEquals(1, expanded.getExpansion().getContains().size()); + assertEquals("http://acme.org", expanded.getExpansion().getContainsFirstRep().getSystem()); + assertEquals("11378-7", expanded.getExpansion().getContainsFirstRep().getCode()); + assertEquals("Systolic blood pressure at First encounter", expanded.getExpansion().getContainsFirstRep().getDisplay()); } From 67d363f9e143cd1d8ce43525440d2ac1a61e3d1b Mon Sep 17 00:00:00 2001 From: James Agnew <jamesagnew@gmail.com> Date: Tue, 23 Jun 2020 11:35:10 -0400 Subject: [PATCH 14/22] Add setting to allow validation of reference targets (#1932) * Add setting to allow validation of reference targets * Add changelog * Add changelog * License headers * Plugin version bump * Experiment with Maven build --- azure-pipelines.yml | 10 + .../1932-allow-validation-of-ref-targets.yaml | 5 + .../hapi/fhir/changelog/5_1_0/changes.yaml | 1 + .../fhir/jpa/config/BaseConfigDstu3Plus.java | 9 +- .../validation/JpaFhirInstanceValidator.java | 114 ++++++++++ .../jpa/validation/ValidationSettings.java | 61 ++++++ .../dao/r4/FhirResourceDaoR4ValidateTest.java | 201 ++++++++++++++++++ .../validator/FhirInstanceValidator.java | 29 ++- pom.xml | 4 +- 9 files changed, 420 insertions(+), 14 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1932-allow-validation-of-ref-targets.yaml create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/JpaFhirInstanceValidator.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/ValidationSettings.java diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5aecb406cba..507a68d4f0b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,6 +25,16 @@ jobs: inputs: targetType: 'inline' script: mkdir -p $(MAVEN_CACHE_FOLDER); pwd; ls -al $(MAVEN_CACHE_FOLDER) + - task: Maven@3 + env: + JAVA_HOME_11_X64: /usr/local/openjdk-11 + inputs: + goals: 'dependency:go-offline' + # These are Maven CLI options (and show up in the build logs) - "-nsu"=Don't update snapshots. We can remove this when Maven OSS is more healthy + options: '-P ALLMODULES,JACOCO,CI,ERRORPRONE -e -B -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + # These are JVM options (and don't show up in the build logs) + mavenOptions: '-Xmx1024m $(MAVEN_OPTS) -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS -Duser.timezone=America/Toronto' + jdkVersionOption: 1.11 - task: Maven@3 env: JAVA_HOME_11_X64: /usr/local/openjdk-11 diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1932-allow-validation-of-ref-targets.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1932-allow-validation-of-ref-targets.yaml new file mode 100644 index 00000000000..e679d4b7e7c --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1932-allow-validation-of-ref-targets.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 1932 +title: "A new configuration bean called ValidationSettings has been added to the JPA server. This can be used + to enable validation of reference target resources if necessary." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/changes.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/changes.yaml index 7bbbb447127..28ad0d1fe6c 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/changes.yaml +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/changes.yaml @@ -4,6 +4,7 @@ title: "The version of a few dependencies have been bumped to the latest versions (dependent HAPI modules listed in brackets): <ul> + <li>Guava (JPA): 28.2-jre -> 29.0-jre</li> </ul>" - item: issue: "1862" diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java index f67f43c0c48..46a56e1aedf 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java @@ -32,7 +32,9 @@ import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvc; import ca.uhn.fhir.jpa.term.api.ITermReindexingSvc; import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc; +import ca.uhn.fhir.jpa.validation.JpaFhirInstanceValidator; import ca.uhn.fhir.jpa.validation.JpaValidationSupportChain; +import ca.uhn.fhir.jpa.validation.ValidationSettings; import ca.uhn.fhir.validation.IInstanceValidatorModule; import org.hl7.fhir.common.hapi.validation.support.CachingValidationSupport; import org.hl7.fhir.common.hapi.validation.support.ValidationSupportChain; @@ -88,12 +90,17 @@ public abstract class BaseConfigDstu3Plus extends BaseConfig { @Bean(name = "myInstanceValidator") @Lazy public IInstanceValidatorModule instanceValidator() { - FhirInstanceValidator val = new FhirInstanceValidator(fhirContext()); + FhirInstanceValidator val = new JpaFhirInstanceValidator(fhirContext()); val.setBestPracticeWarningLevel(IResourceValidator.BestPracticeWarningLevel.Warning); val.setValidationSupport(validationSupportChain()); return val; } + @Bean + public ValidationSettings validationSettings() { + return new ValidationSettings(); + } + @Bean public abstract ITermReadSvc terminologyService(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/JpaFhirInstanceValidator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/JpaFhirInstanceValidator.java new file mode 100644 index 00000000000..d9d4c4351fd --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/JpaFhirInstanceValidator.java @@ -0,0 +1,114 @@ +package ca.uhn.fhir.jpa.validation; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; +import org.hl7.fhir.exceptions.DefinitionException; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.exceptions.FHIRFormatError; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r5.elementmodel.Element; +import org.hl7.fhir.r5.elementmodel.JsonParser; +import org.hl7.fhir.r5.utils.IResourceValidator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Locale; + +public class JpaFhirInstanceValidator extends FhirInstanceValidator { + + private static final Logger ourLog = LoggerFactory.getLogger(JpaFhirInstanceValidator.class); + private final FhirContext myFhirContext; + @Autowired + private ValidationSettings myValidationSettings; + @Autowired + private DaoRegistry myDaoRegistry; + + /** + * Constructor + */ + public JpaFhirInstanceValidator(FhirContext theFhirContext) { + super(theFhirContext); + myFhirContext = theFhirContext; + setValidatorResourceFetcher(new MyValidatorResourceFetcher()); + } + + private class MyValidatorResourceFetcher implements IResourceValidator.IValidatorResourceFetcher { + + + @SuppressWarnings("ConstantConditions") + @Override + public Element fetch(Object appContext, String theUrl) throws FHIRException { + + IdType id = new IdType(theUrl); + String resourceType = id.getResourceType(); + IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceType); + IBaseResource target; + try { + target = dao.read(id, (RequestDetails) appContext); + } catch (ResourceNotFoundException e) { + ourLog.info("Failed to resolve local reference: {}", theUrl); + return null; + } + + try { + return new JsonParser(provideWorkerContext()).parse(myFhirContext.newJsonParser().encodeResourceToString(target), resourceType); + } catch (Exception e) { + throw new FHIRException(e); + } + } + + @Override + public IResourceValidator.ReferenceValidationPolicy validationPolicy(Object appContext, String path, String url) { + int slashIdx = url.indexOf("/"); + if (slashIdx > 0 && myFhirContext.getResourceTypes().contains(url.substring(0, slashIdx))) { + return myValidationSettings.getLocalReferenceValidationDefaultPolicy(); + } + + return IResourceValidator.ReferenceValidationPolicy.IGNORE; + } + + @Override + public boolean resolveURL(Object appContext, String path, String url) throws IOException, FHIRException { + return true; + } + + @Override + public byte[] fetchRaw(String url) throws IOException { + return new byte[0]; + } + + @Override + public void setLocale(Locale locale) { + // ignore + } + + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/ValidationSettings.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/ValidationSettings.java new file mode 100644 index 00000000000..6ca71e20b11 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/ValidationSettings.java @@ -0,0 +1,61 @@ +package ca.uhn.fhir.jpa.validation; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.hl7.fhir.r5.utils.IResourceValidator; +import org.thymeleaf.util.Validate; + +import javax.annotation.Nonnull; + +public class ValidationSettings { + + private IResourceValidator.ReferenceValidationPolicy myLocalReferenceValidationDefaultPolicy = IResourceValidator.ReferenceValidationPolicy.IGNORE; + + /** + * Supplies a default policy for validating local references. Default is {@literal IResourceValidator.ReferenceValidationPolicy.IGNORE}. + * <p> + * Note that this setting can have a measurable impact on validation performance, as it will cause reference targets + * to be resolved during validation. In other words, if a resource has a reference to (for example) "Patient/123", the + * resource with that ID will be loaded from the database during validation. + * </p> + * + * @since 5.1.0 + */ + @Nonnull + public IResourceValidator.ReferenceValidationPolicy getLocalReferenceValidationDefaultPolicy() { + return myLocalReferenceValidationDefaultPolicy; + } + + /** + * Supplies a default policy for validating local references. Default is {@literal IResourceValidator.ReferenceValidationPolicy.IGNORE}. + * <p> + * Note that this setting can have a measurable impact on validation performance, as it will cause reference targets + * to be resolved during validation. In other words, if a resource has a reference to (for example) "Patient/123", the + * resource with that ID will be loaded from the database during validation. + * </p> + * + * @since 5.1.0 + */ + public void setLocalReferenceValidationDefaultPolicy(@Nonnull IResourceValidator.ReferenceValidationPolicy theLocalReferenceValidationDefaultPolicy) { + Validate.notNull(theLocalReferenceValidationDefaultPolicy, "theLocalReferenceValidationDefaultPolicy must not be null"); + myLocalReferenceValidationDefaultPolicy = theLocalReferenceValidationDefaultPolicy; + } +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java index cd38f6ab409..24327236d2e 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4ValidateTest.java @@ -11,6 +11,7 @@ import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc; import ca.uhn.fhir.jpa.term.api.ITermReadSvc; import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet; import ca.uhn.fhir.jpa.validation.JpaValidationSupportChain; +import ca.uhn.fhir.jpa.validation.ValidationSettings; import ca.uhn.fhir.rest.api.EncodingEnum; import ca.uhn.fhir.rest.api.MethodOutcome; import ca.uhn.fhir.rest.api.ValidationModeEnum; @@ -35,6 +36,7 @@ import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Condition; import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.Group; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Narrative; import org.hl7.fhir.r4.model.Observation; @@ -42,6 +44,7 @@ import org.hl7.fhir.r4.model.Observation.ObservationStatus; import org.hl7.fhir.r4.model.OperationOutcome; import org.hl7.fhir.r4.model.Organization; import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Practitioner; import org.hl7.fhir.r4.model.Questionnaire; import org.hl7.fhir.r4.model.QuestionnaireResponse; import org.hl7.fhir.r4.model.Reference; @@ -86,6 +89,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { private JpaValidationSupportChain myJpaValidationSupportChain; @Autowired private PlatformTransactionManager myTransactionManager; + @Autowired + private ValidationSettings myValidationSettings; /** * Create a loinc valueset that expands to more results than the expander is willing to do @@ -187,6 +192,200 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { } + @Test + public void testValidateProfileTargetType_PolicyCheckValid() throws IOException { + myValidationSettings.setLocalReferenceValidationDefaultPolicy(IResourceValidator.ReferenceValidationPolicy.CHECK_VALID); + + StructureDefinition profile = loadResourceFromClasspath(StructureDefinition.class, "/r4/profile-vitalsigns-all-loinc.json"); + myStructureDefinitionDao.create(profile, mySrd); + + ValueSet vs = new ValueSet(); + vs.setUrl("http://example.com/fhir/ValueSet/observation-vitalsignresult"); + vs.getCompose().addInclude().setSystem("http://loinc.org"); + myValueSetDao.create(vs); + + CodeSystem cs = new CodeSystem(); + cs.setContent(CodeSystem.CodeSystemContentMode.COMPLETE); + cs.setUrl("http://loinc.org"); + cs.addConcept().setCode("123-4").setDisplay("Code 123 4"); + myCodeSystemDao.create(cs); + + Group group = new Group(); + group.setId("ABC"); + group.setActive(true); + myGroupDao.update(group); + + Patient patient = new Patient(); + patient.setId("DEF"); + patient.setActive(true); + myPatientDao.update(patient); + + Practitioner practitioner = new Practitioner(); + practitioner.setId("P"); + practitioner.setActive(true); + myPractitionerDao.update(practitioner); + + Observation obs = new Observation(); + obs.getMeta().addProfile("http://example.com/fhir/StructureDefinition/vitalsigns-2"); + obs.getText().setDivAsString("<div>Hello</div>"); + obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs"); + obs.addPerformer(new Reference("Practitioner/P")); + obs.setEffective(DateTimeType.now()); + obs.setStatus(ObservationStatus.FINAL); + obs.setValue(new StringType("This is the value")); + obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED); + obs.getCode().getCodingFirstRep().setSystem("http://loinc.org").setCode("123-4").setDisplay("Display 3"); + + // Non-existent target + obs.setSubject(new Reference("Group/123")); + OperationOutcome oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "Unable to resolve resource \"Group/123\"", oo.getIssueFirstRep().getDiagnostics()); + + // Target of wrong type + obs.setSubject(new Reference("Group/ABC")); + oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "Invalid Resource target type. Found Group, but expected one of ([Patient])", oo.getIssueFirstRep().getDiagnostics()); + + // Target of right type + obs.setSubject(new Reference("Patient/DEF")); + oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "No issues detected during validation", oo.getIssueFirstRep().getDiagnostics()); + + } + + @Test + public void testValidateProfileTargetType_PolicyCheckExistsAndType() throws IOException { + myValidationSettings.setLocalReferenceValidationDefaultPolicy(IResourceValidator.ReferenceValidationPolicy.CHECK_EXISTS_AND_TYPE); + + StructureDefinition profile = loadResourceFromClasspath(StructureDefinition.class, "/r4/profile-vitalsigns-all-loinc.json"); + myStructureDefinitionDao.create(profile, mySrd); + + ValueSet vs = new ValueSet(); + vs.setUrl("http://example.com/fhir/ValueSet/observation-vitalsignresult"); + vs.getCompose().addInclude().setSystem("http://loinc.org"); + myValueSetDao.create(vs); + + CodeSystem cs = new CodeSystem(); + cs.setContent(CodeSystem.CodeSystemContentMode.COMPLETE); + cs.setUrl("http://loinc.org"); + cs.addConcept().setCode("123-4").setDisplay("Code 123 4"); + myCodeSystemDao.create(cs); + + Group group = new Group(); + group.setId("ABC"); + group.setActive(true); + myGroupDao.update(group); + + Patient patient = new Patient(); + patient.setId("DEF"); + patient.setActive(true); + myPatientDao.update(patient); + + Practitioner practitioner = new Practitioner(); + practitioner.setId("P"); + practitioner.setActive(true); + myPractitionerDao.update(practitioner); + + Observation obs = new Observation(); + obs.getMeta().addProfile("http://example.com/fhir/StructureDefinition/vitalsigns-2"); + obs.getText().setDivAsString("<div>Hello</div>"); + obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs"); + obs.addPerformer(new Reference("Practitioner/P")); + obs.setEffective(DateTimeType.now()); + obs.setStatus(ObservationStatus.FINAL); + obs.setValue(new StringType("This is the value")); + obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED); + obs.getCode().getCodingFirstRep().setSystem("http://loinc.org").setCode("123-4").setDisplay("Display 3"); + + // Non-existent target + obs.setSubject(new Reference("Group/123")); + OperationOutcome oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "Unable to resolve resource \"Group/123\"", oo.getIssueFirstRep().getDiagnostics()); + + // Target of wrong type + obs.setSubject(new Reference("Group/ABC")); + oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "Unable to find matching profile for Group/ABC (by type) among choices: ; [CanonicalType[http://hl7.org/fhir/StructureDefinition/Patient]]", oo.getIssueFirstRep().getDiagnostics()); + + // Target of right type + obs.setSubject(new Reference("Patient/DEF")); + oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "No issues detected during validation", oo.getIssueFirstRep().getDiagnostics()); + + } + + + @Test + public void testValidateProfileTargetType_PolicyCheckExists() throws IOException { + myValidationSettings.setLocalReferenceValidationDefaultPolicy(IResourceValidator.ReferenceValidationPolicy.CHECK_EXISTS); + + StructureDefinition profile = loadResourceFromClasspath(StructureDefinition.class, "/r4/profile-vitalsigns-all-loinc.json"); + myStructureDefinitionDao.create(profile, mySrd); + + ValueSet vs = new ValueSet(); + vs.setUrl("http://example.com/fhir/ValueSet/observation-vitalsignresult"); + vs.getCompose().addInclude().setSystem("http://loinc.org"); + myValueSetDao.create(vs); + + CodeSystem cs = new CodeSystem(); + cs.setContent(CodeSystem.CodeSystemContentMode.COMPLETE); + cs.setUrl("http://loinc.org"); + cs.addConcept().setCode("123-4").setDisplay("Code 123 4"); + myCodeSystemDao.create(cs); + + Group group = new Group(); + group.setId("ABC"); + group.setActive(true); + myGroupDao.update(group); + + Patient patient = new Patient(); + patient.setId("DEF"); + patient.setActive(true); + myPatientDao.update(patient); + + Practitioner practitioner = new Practitioner(); + practitioner.setId("P"); + practitioner.setActive(true); + myPractitionerDao.update(practitioner); + + Observation obs = new Observation(); + obs.getMeta().addProfile("http://example.com/fhir/StructureDefinition/vitalsigns-2"); + obs.getText().setDivAsString("<div>Hello</div>"); + obs.getCategoryFirstRep().addCoding().setSystem("http://terminology.hl7.org/CodeSystem/observation-category").setCode("vital-signs"); + obs.addPerformer(new Reference("Practitioner/P")); + obs.setEffective(DateTimeType.now()); + obs.setStatus(ObservationStatus.FINAL); + obs.setValue(new StringType("This is the value")); + obs.getText().setStatus(Narrative.NarrativeStatus.GENERATED); + obs.getCode().getCodingFirstRep().setSystem("http://loinc.org").setCode("123-4").setDisplay("Display 3"); + + // Non-existent target + obs.setSubject(new Reference("Group/123")); + OperationOutcome oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "Unable to resolve resource \"Group/123\"", oo.getIssueFirstRep().getDiagnostics()); + + // Target of wrong type + obs.setSubject(new Reference("Group/ABC")); + oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "No issues detected during validation", oo.getIssueFirstRep().getDiagnostics()); + + // Target of right type + obs.setSubject(new Reference("Patient/DEF")); + oo = validateAndReturnOutcome(obs); + ourLog.info(myFhirCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(oo)); + assertEquals(encode(oo), "No issues detected during validation", oo.getIssueFirstRep().getDiagnostics()); + + } + + /** * Per: https://chat.fhir.org/#narrow/stream/179166-implementers/topic/Handling.20incomplete.20CodeSystems * <p> @@ -743,6 +942,8 @@ public class FhirResourceDaoR4ValidateTest extends BaseJpaR4Test { myDaoConfig.setPreExpandValueSets(new DaoConfig().isPreExpandValueSets()); BaseTermReadSvcImpl.setInvokeOnNextCallForUnitTest(null); + + myValidationSettings.setLocalReferenceValidationDefaultPolicy(IResourceValidator.ReferenceValidationPolicy.IGNORE); } @Test diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java index 41d5b44f6e6..605b01282c7 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/validator/FhirInstanceValidator.java @@ -21,6 +21,7 @@ import org.hl7.fhir.r5.utils.IResourceValidator; import org.hl7.fhir.r5.utils.IResourceValidator.BestPracticeWarningLevel; import org.hl7.fhir.utilities.validation.ValidationMessage; +import javax.annotation.Nonnull; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -197,6 +198,21 @@ public class FhirInstanceValidator extends BaseValidatorBridge implements IInsta @Override protected List<ValidationMessage> validate(IValidationContext<?> theValidationCtx) { + VersionSpecificWorkerContextWrapper wrappedWorkerContext = provideWorkerContext(); + + return new ValidatorWrapper() + .setAnyExtensionsAllowed(isAnyExtensionsAllowed()) + .setBestPracticeWarningLevel(getBestPracticeWarningLevel()) + .setErrorForUnknownProfiles(isErrorForUnknownProfiles()) + .setExtensionDomains(getExtensionDomains()) + .setNoTerminologyChecks(isNoTerminologyChecks()) + .setValidatorResourceFetcher(getValidatorResourceFetcher()) + .setAssumeValidRestReferences(isAssumeValidRestReferences()) + .validate(wrappedWorkerContext, theValidationCtx); + } + + @Nonnull + protected VersionSpecificWorkerContextWrapper provideWorkerContext() { VersionSpecificWorkerContextWrapper wrappedWorkerContext = myWrappedWorkerContext; if (wrappedWorkerContext == null) { VersionSpecificWorkerContextWrapper.IVersionTypeConverter converter; @@ -213,7 +229,7 @@ public class FhirInstanceValidator extends BaseValidatorBridge implements IInsta org.hl7.fhir.dstu2.model.ValueSet valueSet = (org.hl7.fhir.dstu2.model.ValueSet) nonCanonical; if (valueSet.hasCodeSystem() && valueSet.getCodeSystem().hasSystem()) { if (!valueSet.hasCompose()) { - org.hl7.fhir.r5.model.ValueSet valueSetR5 = (org.hl7.fhir.r5.model.ValueSet) retVal; + ValueSet valueSetR5 = (ValueSet) retVal; valueSetR5.getCompose().addInclude().setSystem(valueSet.getCodeSystem().getSystem()); } } @@ -257,16 +273,7 @@ public class FhirInstanceValidator extends BaseValidatorBridge implements IInsta wrappedWorkerContext = new VersionSpecificWorkerContextWrapper(new ValidationSupportContext(myValidationSupport), converter); } myWrappedWorkerContext = wrappedWorkerContext; - - return new ValidatorWrapper() - .setAnyExtensionsAllowed(isAnyExtensionsAllowed()) - .setBestPracticeWarningLevel(getBestPracticeWarningLevel()) - .setErrorForUnknownProfiles(isErrorForUnknownProfiles()) - .setExtensionDomains(getExtensionDomains()) - .setNoTerminologyChecks(isNoTerminologyChecks()) - .setValidatorResourceFetcher(getValidatorResourceFetcher()) - .setAssumeValidRestReferences(isAssumeValidRestReferences()) - .validate(wrappedWorkerContext, theValidationCtx); + return wrappedWorkerContext; } private FhirContext getDstu2Context() { diff --git a/pom.xml b/pom.xml index 9cadc72dcbf..5f6664c3e5d 100644 --- a/pom.xml +++ b/pom.xml @@ -693,7 +693,7 @@ <!--<derby_version>10.15.1.3</derby_version>--> <error_prone_annotations_version>2.3.4</error_prone_annotations_version> <error_prone_core_version>2.3.3</error_prone_core_version> - <guava_version>28.2-jre</guava_version> + <guava_version>29.0-jre</guava_version> <gson_version>2.8.5</gson_version> <jaxb_bundle_version>2.2.11_1</jaxb_bundle_version> <jaxb_api_version>2.3.1</jaxb_api_version> @@ -1669,7 +1669,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> - <version>1.8</version> + <version>3.0.0</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> From f88298a1fbe100a6e2d5946b7e9d96698f8f3d42 Mon Sep 17 00:00:00 2001 From: James Agnew <jamesagnew@gmail.com> Date: Tue, 23 Jun 2020 11:35:26 -0400 Subject: [PATCH 15/22] Remote terminology service enhancements (#1934) * Remote terminology service enhancements * Add changelog --- .../DefaultProfileValidationSupport.java | 20 +- .../1934-remote-termsvc-enhancements.yaml | 8 + .../validation/validation_support_modules.md | 6 + ...teTerminologyServiceValidationSupport.java | 108 ++++++++-- ...rminologyServiceValidationSupportTest.java | 198 +++++++++++++++--- 5 files changed, 283 insertions(+), 57 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1934-remote-termsvc-enhancements.yaml diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/DefaultProfileValidationSupport.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/DefaultProfileValidationSupport.java index bf03f54eb1d..c8e7804fc63 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/DefaultProfileValidationSupport.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/DefaultProfileValidationSupport.java @@ -31,6 +31,7 @@ import org.hl7.fhir.instance.model.api.IBaseBundle; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IPrimitiveType; +import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -327,13 +328,7 @@ public class DefaultProfileValidationSupport implements IValidationSupport { } private String getConformanceResourceUrl(IBaseResource theResource) { - String urlValueString = null; - Optional<IBase> urlValue = getFhirContext().getResourceDefinition(theResource).getChildByName("url").getAccessor().getFirstValueOrNull(theResource); - if (urlValue.isPresent()) { - IPrimitiveType<?> urlValueType = (IPrimitiveType<?>) urlValue.get(); - urlValueString = urlValueType.getValueAsString(); - } - return urlValueString; + return getConformanceResourceUrl(getFhirContext(), theResource); } private List<IBaseResource> parseBundle(InputStreamReader theReader) { @@ -346,6 +341,17 @@ public class DefaultProfileValidationSupport implements IValidationSupport { } } + @Nullable + public static String getConformanceResourceUrl(FhirContext theFhirContext, IBaseResource theResource) { + String urlValueString = null; + Optional<IBase> urlValue = theFhirContext.getResourceDefinition(theResource).getChildByName("url").getAccessor().getFirstValueOrNull(theResource); + if (urlValue.isPresent()) { + IPrimitiveType<?> urlValueType = (IPrimitiveType<?>) urlValue.get(); + urlValueString = urlValueType.getValueAsString(); + } + return urlValueString; + } + static <T extends IBaseResource> List<T> toList(Map<String, IBaseResource> theMap) { ArrayList<IBaseResource> retVal = new ArrayList<>(theMap.values()); return (List<T>) Collections.unmodifiableList(retVal); diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1934-remote-termsvc-enhancements.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1934-remote-termsvc-enhancements.yaml new file mode 100644 index 00000000000..8a7a29ccd5b --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1934-remote-termsvc-enhancements.yaml @@ -0,0 +1,8 @@ +--- +type: add +issue: 1934 +title: "The **RemoteTerminologyServiceValidationSupport** validation support module, which is used to connect to + external/remote terminology services, has been significantly enhanced to provide testing for supported + CodeSystems and ValueSets. It will also now validate codes in fields that are not bound to a specific + ValueSet." + diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/validation/validation_support_modules.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/validation/validation_support_modules.md index afffbe8a490..cdd07b8bd39 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/validation/validation_support_modules.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/validation/validation_support_modules.md @@ -118,6 +118,12 @@ The following table lists vocabulary that is validated by this module: This module validates codes using a remote FHIR-based terminology server. +This module will invoke the following operations on the remote terminology server: + +* **GET [base]/CodeSystem?url=[url]** – Tests whether a given CodeSystem is supported on the server +* **GET [base]/ValueSet?url=[url]** – Tests whether a given ValueSet is supported on the server +* **POST [base]/CodeSystem/$validate-code** – Validate codes in fields where no specific ValueSet is bound +* **POST [base]/ValueSet/$validate-code** – Validate codes in fields where a specific ValueSet is bound # Recipes diff --git a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupport.java b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupport.java index 619512b1953..5d62a28d619 100644 --- a/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupport.java +++ b/hapi-fhir-validation/src/main/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupport.java @@ -2,13 +2,17 @@ package org.hl7.fhir.common.hapi.validation.support; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.ConceptValidationOptions; +import ca.uhn.fhir.context.support.DefaultProfileValidationSupport; import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.context.support.ValidationSupportContext; import ca.uhn.fhir.rest.client.api.IGenericClient; +import ca.uhn.fhir.util.BundleUtil; import ca.uhn.fhir.util.ParametersUtil; import org.apache.commons.lang3.Validate; +import org.hl7.fhir.instance.model.api.IBaseBundle; import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeSystem; import javax.annotation.Nonnull; import java.util.ArrayList; @@ -42,6 +46,64 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup return invokeRemoteValidateCode(theCodeSystem, theCode, theDisplay, theValueSetUrl, null); } + @Override + public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) { + IBaseResource valueSet = theValueSet; + String valueSetUrl = DefaultProfileValidationSupport.getConformanceResourceUrl(myCtx, valueSet); + if (isNotBlank(valueSetUrl)) { + valueSet = null; + } else { + valueSetUrl = null; + } + return invokeRemoteValidateCode(theCodeSystem, theCode, theDisplay, valueSetUrl, valueSet); + } + + @Override + public IBaseResource fetchCodeSystem(String theSystem) { + IGenericClient client = provideClient(); + Class<? extends IBaseBundle> bundleType = myCtx.getResourceDefinition("Bundle").getImplementingClass(IBaseBundle.class); + IBaseBundle results = client + .search() + .forResource("CodeSystem") + .where(CodeSystem.URL.matches().value(theSystem)) + .returnBundle(bundleType) + .execute(); + List<IBaseResource> resultsList = BundleUtil.toListOfResources(myCtx, results); + if (resultsList.size() > 0) { + return resultsList.get(0); + } + + return null; + } + + @Override + public IBaseResource fetchValueSet(String theValueSetUrl) { + IGenericClient client = provideClient(); + Class<? extends IBaseBundle> bundleType = myCtx.getResourceDefinition("Bundle").getImplementingClass(IBaseBundle.class); + IBaseBundle results = client + .search() + .forResource("ValueSet") + .where(CodeSystem.URL.matches().value(theValueSetUrl)) + .returnBundle(bundleType) + .execute(); + List<IBaseResource> resultsList = BundleUtil.toListOfResources(myCtx, results); + if (resultsList.size() > 0) { + return resultsList.get(0); + } + + return null; + } + + @Override + public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) { + return fetchCodeSystem(theSystem) != null; + } + + @Override + public boolean isValueSetSupported(ValidationSupportContext theValidationSupportContext, String theValueSetUrl) { + return fetchValueSet(theValueSetUrl) != null; + } + private IGenericClient provideClient() { IGenericClient retVal = myCtx.newRestfulGenericClient(myBaseUrl); for (Object next : myClientInterceptors) { @@ -50,11 +112,6 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup return retVal; } - @Override - public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) { - return invokeRemoteValidateCode(theCodeSystem, theCode, theDisplay, null, theValueSet); - } - protected CodeValidationResult invokeRemoteValidateCode(String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl, IBaseResource theValueSet) { if (isBlank(theCode)) { return null; @@ -64,23 +121,38 @@ public class RemoteTerminologyServiceValidationSupport extends BaseValidationSup IBaseParameters input = ParametersUtil.newInstance(getFhirContext()); - if (isNotBlank(theValueSetUrl)) { - ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "url", theValueSetUrl); - } - ParametersUtil.addParameterToParametersString(getFhirContext(), input, "code", theCode); - if (isNotBlank(theCodeSystem)) { - ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "system", theCodeSystem); - } - if (isNotBlank(theDisplay)) { - ParametersUtil.addParameterToParametersString(getFhirContext(), input, "display", theDisplay); - } - if (theValueSet != null) { - ParametersUtil.addParameterToParameters(getFhirContext(), input, "valueSet", theValueSet); + String resourceType = "ValueSet"; + if (theValueSet == null && theValueSetUrl == null) { + resourceType = "CodeSystem"; + + ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "url", theCodeSystem); + ParametersUtil.addParameterToParametersString(getFhirContext(), input, "code", theCode); + if (isNotBlank(theDisplay)) { + ParametersUtil.addParameterToParametersString(getFhirContext(), input, "display", theDisplay); + } + + } else { + + if (isNotBlank(theValueSetUrl)) { + ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "url", theValueSetUrl); + } + ParametersUtil.addParameterToParametersString(getFhirContext(), input, "code", theCode); + if (isNotBlank(theCodeSystem)) { + ParametersUtil.addParameterToParametersUri(getFhirContext(), input, "system", theCodeSystem); + } + if (isNotBlank(theDisplay)) { + ParametersUtil.addParameterToParametersString(getFhirContext(), input, "display", theDisplay); + } + if (theValueSet != null) { + ParametersUtil.addParameterToParameters(getFhirContext(), input, "valueSet", theValueSet); + } + } + IBaseParameters output = client .operation() - .onType("ValueSet") + .onType(resourceType) .named("validate-code") .withParameters(input) .execute(); diff --git a/hapi-fhir-validation/src/test/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupportTest.java b/hapi-fhir-validation/src/test/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupportTest.java index dda79bab7b4..6d14bb0a3da 100644 --- a/hapi-fhir-validation/src/test/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupportTest.java +++ b/hapi-fhir-validation/src/test/java/org/hl7/fhir/common/hapi/validation/support/RemoteTerminologyServiceValidationSupportTest.java @@ -5,9 +5,15 @@ import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.rest.annotation.IdParam; import ca.uhn.fhir.rest.annotation.Operation; import ca.uhn.fhir.rest.annotation.OperationParam; +import ca.uhn.fhir.rest.annotation.RequiredParam; +import ca.uhn.fhir.rest.annotation.Search; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; +import ca.uhn.fhir.rest.param.UriParam; +import ca.uhn.fhir.rest.server.IResourceProvider; import ca.uhn.fhir.test.utilities.server.RestfulServerRule; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.BooleanType; +import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Parameters; @@ -21,6 +27,9 @@ import org.junit.Test; import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; + import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -37,13 +46,18 @@ public class RemoteTerminologyServiceValidationSupportTest { @Rule public RestfulServerRule myRestfulServerRule = new RestfulServerRule(ourCtx); - private MyMockTerminologyServiceProvider myProvider; + private MyValueSetProvider myValueSetProvider; private RemoteTerminologyServiceValidationSupport mySvc; + private MyCodeSystemProvider myCodeSystemProvider; @Before public void before() { - myProvider = new MyMockTerminologyServiceProvider(); - myRestfulServerRule.getRestfulServer().registerProvider(myProvider); + myValueSetProvider = new MyValueSetProvider(); + myRestfulServerRule.getRestfulServer().registerProvider(myValueSetProvider); + + myCodeSystemProvider = new MyCodeSystemProvider(); + myRestfulServerRule.getRestfulServer().registerProvider(myCodeSystemProvider); + String baseUrl = "http://localhost:" + myRestfulServerRule.getPort(); mySvc = new RemoteTerminologyServiceValidationSupport(ourCtx); @@ -53,7 +67,7 @@ public class RemoteTerminologyServiceValidationSupportTest { @After public void after() { - assertThat(myProvider.myInvocationCount, lessThan(2)); + assertThat(myValueSetProvider.myInvocationCount, lessThan(2)); } @Test @@ -64,7 +78,7 @@ public class RemoteTerminologyServiceValidationSupportTest { @Test public void testValidateCode_SystemCodeDisplayUrl_Success() { - createNextReturnParameters(true, DISPLAY, null); + createNextValueSetReturnParameters(true, DISPLAY, null); IValidationSupport.CodeValidationResult outcome = mySvc.validateCode(null, null, CODE_SYSTEM, CODE, DISPLAY, VALUE_SET_URL); assertEquals(CODE, outcome.getCode()); @@ -72,16 +86,16 @@ public class RemoteTerminologyServiceValidationSupportTest { assertEquals(null, outcome.getSeverity()); assertEquals(null, outcome.getMessage()); - assertEquals(CODE, myProvider.myLastCode.getCode()); - assertEquals(DISPLAY, myProvider.myLastDisplay.getValue()); - assertEquals(CODE_SYSTEM, myProvider.myLastSystem.getValue()); - assertEquals(VALUE_SET_URL, myProvider.myLastUrl.getValue()); - assertEquals(null, myProvider.myLastValueSet); + assertEquals(CODE, myValueSetProvider.myLastCode.getCode()); + assertEquals(DISPLAY, myValueSetProvider.myLastDisplay.getValue()); + assertEquals(CODE_SYSTEM, myValueSetProvider.myLastSystem.getValue()); + assertEquals(VALUE_SET_URL, myValueSetProvider.myLastUrl.getValue()); + assertEquals(null, myValueSetProvider.myLastValueSet); } @Test public void testValidateCode_SystemCodeDisplayUrl_Error() { - createNextReturnParameters(false, null, ERROR_MESSAGE); + createNextValueSetReturnParameters(false, null, ERROR_MESSAGE); IValidationSupport.CodeValidationResult outcome = mySvc.validateCode(null, null, CODE_SYSTEM, CODE, DISPLAY, VALUE_SET_URL); assertEquals(null, outcome.getCode()); @@ -89,16 +103,32 @@ public class RemoteTerminologyServiceValidationSupportTest { assertEquals(IValidationSupport.IssueSeverity.ERROR, outcome.getSeverity()); assertEquals(ERROR_MESSAGE, outcome.getMessage()); - assertEquals(CODE, myProvider.myLastCode.getCode()); - assertEquals(DISPLAY, myProvider.myLastDisplay.getValue()); - assertEquals(CODE_SYSTEM, myProvider.myLastSystem.getValue()); - assertEquals(VALUE_SET_URL, myProvider.myLastUrl.getValue()); - assertEquals(null, myProvider.myLastValueSet); + assertEquals(CODE, myValueSetProvider.myLastCode.getCode()); + assertEquals(DISPLAY, myValueSetProvider.myLastDisplay.getValue()); + assertEquals(CODE_SYSTEM, myValueSetProvider.myLastSystem.getValue()); + assertEquals(VALUE_SET_URL, myValueSetProvider.myLastUrl.getValue()); + assertEquals(null, myValueSetProvider.myLastValueSet); } + @Test + public void testValidateCodeInCodeSystem_Good() { + createNextCodeSystemReturnParameters(true, DISPLAY, null); + + IValidationSupport.CodeValidationResult outcome = mySvc.validateCode(null, null, CODE_SYSTEM, CODE, DISPLAY, null); + assertEquals(CODE, outcome.getCode()); + assertEquals(DISPLAY, outcome.getDisplay()); + assertEquals(null, outcome.getSeverity()); + assertEquals(null, outcome.getMessage()); + + assertEquals(CODE, myCodeSystemProvider.myLastCode.getCode()); + assertEquals(DISPLAY, myCodeSystemProvider.myLastDisplay.getValue()); + assertEquals(CODE_SYSTEM, myCodeSystemProvider.myLastUrl.getValueAsString()); + } + + @Test public void testValidateCodeInValueSet_SystemCodeDisplayVS_Good() { - createNextReturnParameters(true, DISPLAY, null); + createNextValueSetReturnParameters(true, DISPLAY, null); ValueSet valueSet = new ValueSet(); valueSet.setUrl(VALUE_SET_URL); @@ -109,34 +139,127 @@ public class RemoteTerminologyServiceValidationSupportTest { assertEquals(null, outcome.getSeverity()); assertEquals(null, outcome.getMessage()); - assertEquals(CODE, myProvider.myLastCode.getCode()); - assertEquals(DISPLAY, myProvider.myLastDisplay.getValue()); - assertEquals(CODE_SYSTEM, myProvider.myLastSystem.getValue()); - assertEquals(null, myProvider.myLastUrl); - assertEquals(VALUE_SET_URL, myProvider.myLastValueSet.getUrl()); + assertEquals(CODE, myValueSetProvider.myLastCode.getCode()); + assertEquals(DISPLAY, myValueSetProvider.myLastDisplay.getValue()); + assertEquals(CODE_SYSTEM, myValueSetProvider.myLastSystem.getValue()); + assertEquals(VALUE_SET_URL, myValueSetProvider.myLastUrl.getValueAsString()); + assertEquals(null, myValueSetProvider.myLastValueSet); } - public void createNextReturnParameters(boolean theResult, String theDisplay, String theMessage) { - myProvider.myNextReturn = new Parameters(); - myProvider.myNextReturn.addParameter("result", theResult); - myProvider.myNextReturn.addParameter("display", theDisplay); + @Test + public void testIsValueSetSupported_False() { + myValueSetProvider.myNextReturnValueSets = new ArrayList<>(); + + boolean outcome = mySvc.isValueSetSupported(null, "http://loinc.org/VS"); + assertEquals(false, outcome); + assertEquals("http://loinc.org/VS", myValueSetProvider.myLastUrlParam.getValue()); + } + + @Test + public void testIsValueSetSupported_True() { + myValueSetProvider.myNextReturnValueSets = new ArrayList<>(); + myValueSetProvider.myNextReturnValueSets.add((ValueSet) new ValueSet().setId("ValueSet/123")); + + boolean outcome = mySvc.isValueSetSupported(null, "http://loinc.org/VS"); + assertEquals(true, outcome); + assertEquals("http://loinc.org/VS", myValueSetProvider.myLastUrlParam.getValue()); + } + + @Test + public void testIsCodeSystemSupported_False() { + myCodeSystemProvider.myNextReturnCodeSystems = new ArrayList<>(); + + boolean outcome = mySvc.isCodeSystemSupported(null, "http://loinc.org"); + assertEquals(false, outcome); + assertEquals("http://loinc.org", myCodeSystemProvider.myLastUrlParam.getValue()); + } + + @Test + public void testIsCodeSystemSupported_True() { + myCodeSystemProvider.myNextReturnCodeSystems = new ArrayList<>(); + myCodeSystemProvider.myNextReturnCodeSystems.add((CodeSystem) new CodeSystem().setId("CodeSystem/123")); + + boolean outcome = mySvc.isCodeSystemSupported(null, "http://loinc.org"); + assertEquals(true, outcome); + assertEquals("http://loinc.org", myCodeSystemProvider.myLastUrlParam.getValue()); + } + + private void createNextCodeSystemReturnParameters(boolean theResult, String theDisplay, String theMessage) { + myCodeSystemProvider.myNextReturnParams = new Parameters(); + myCodeSystemProvider.myNextReturnParams.addParameter("result", theResult); + myCodeSystemProvider.myNextReturnParams.addParameter("display", theDisplay); if (theMessage != null) { - myProvider.myNextReturn.addParameter("message", theMessage); + myCodeSystemProvider.myNextReturnParams.addParameter("message", theMessage); } } - private static class MyMockTerminologyServiceProvider { + private void createNextValueSetReturnParameters(boolean theResult, String theDisplay, String theMessage) { + myValueSetProvider.myNextReturnParams = new Parameters(); + myValueSetProvider.myNextReturnParams.addParameter("result", theResult); + myValueSetProvider.myNextReturnParams.addParameter("display", theDisplay); + if (theMessage != null) { + myValueSetProvider.myNextReturnParams.addParameter("message", theMessage); + } + } + + private static class MyCodeSystemProvider implements IResourceProvider { + + private UriParam myLastUrlParam; + private List<CodeSystem> myNextReturnCodeSystems; + private int myInvocationCount; + private UriType myLastUrl; + private CodeType myLastCode; + private StringType myLastDisplay; + private Parameters myNextReturnParams; + + @Operation(name = "validate-code", idempotent = true, returnParameters = { + @OperationParam(name = "result", type = BooleanType.class, min = 1), + @OperationParam(name = "message", type = StringType.class), + @OperationParam(name = "display", type = StringType.class) + }) + public Parameters validateCode( + HttpServletRequest theServletRequest, + @IdParam(optional = true) IdType theId, + @OperationParam(name = "url", min = 0, max = 1) UriType theCodeSystemUrl, + @OperationParam(name = "code", min = 0, max = 1) CodeType theCode, + @OperationParam(name = "display", min = 0, max = 1) StringType theDisplay + ) { + myInvocationCount++; + myLastUrl = theCodeSystemUrl; + myLastCode = theCode; + myLastDisplay = theDisplay; + return myNextReturnParams; + + } + + @Search + public List<CodeSystem> find(@RequiredParam(name="url") UriParam theUrlParam) { + myLastUrlParam = theUrlParam; + assert myNextReturnCodeSystems != null; + return myNextReturnCodeSystems; + } + + @Override + public Class<? extends IBaseResource> getResourceType() { + return CodeSystem.class; + } + } - private Parameters myNextReturn; + private static class MyValueSetProvider implements IResourceProvider { + + + private Parameters myNextReturnParams; + private List<ValueSet> myNextReturnValueSets; private UriType myLastUrl; private CodeType myLastCode; private int myInvocationCount; private UriType myLastSystem; private StringType myLastDisplay; private ValueSet myLastValueSet; + private UriParam myLastUrlParam; - @Operation(name = "validate-code", idempotent = true, typeName = "ValueSet", returnParameters = { + @Operation(name = "validate-code", idempotent = true, returnParameters = { @OperationParam(name = "result", type = BooleanType.class, min = 1), @OperationParam(name = "message", type = StringType.class), @OperationParam(name = "display", type = StringType.class) @@ -156,11 +279,22 @@ public class RemoteTerminologyServiceValidationSupportTest { myLastSystem = theSystem; myLastDisplay = theDisplay; myLastValueSet = theValueSet; - return myNextReturn; - + return myNextReturnParams; } + @Search + public List<ValueSet> find(@RequiredParam(name="url") UriParam theUrlParam) { + myLastUrlParam = theUrlParam; + assert myNextReturnValueSets != null; + return myNextReturnValueSets; + } + + @Override + public Class<? extends IBaseResource> getResourceType() { + return ValueSet.class; + } } + } From 6825d2fcf0342b945814a4a695cba1a51e716f47 Mon Sep 17 00:00:00 2001 From: James Agnew <jamesagnew@gmail.com> Date: Tue, 23 Jun 2020 16:26:04 -0400 Subject: [PATCH 16/22] Reduce DB roundtrips for revincludes (#1937) * Reduce DB roundtrips for revincludes * Add changelog --- .../1937-reduce-revinclude-db-roundtrips.yaml | 6 +++ .../fhir/jpa/dao/data/IResourceTagDao.java | 2 +- .../r4/FhirResourceDaoR4QueryCountTest.java | 49 +++++++++++++++++++ .../fhir/jpa/model/entity/ResourceTag.java | 2 +- .../jpa/searchparam/SearchParameterMap.java | 6 ++- 5 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1937-reduce-revinclude-db-roundtrips.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1937-reduce-revinclude-db-roundtrips.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1937-reduce-revinclude-db-roundtrips.yaml new file mode 100644 index 00000000000..fdecae9c2d2 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1937-reduce-revinclude-db-roundtrips.yaml @@ -0,0 +1,6 @@ +--- +type: perf +issue: 1937 +title: Due to an inefficient SQL statement when performing searches with large numbers of _revincludes where the resources + have tags, a large number of database roundtrips were produced when searching. This has been streamlined, greatly improving + the response times for some searches. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTagDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTagDao.java index bb81b65ce0a..e565db8c193 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTagDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceTagDao.java @@ -32,7 +32,7 @@ import ca.uhn.fhir.jpa.model.entity.ResourceTag; public interface IResourceTagDao extends JpaRepository<ResourceTag, Long> { @Query("" + "SELECT t FROM ResourceTag t " + - "INNER JOIN TagDefinition td ON (td.myId = t.myTagId) " + + "INNER JOIN FETCH t.myTag td " + "WHERE t.myResourceId in (:pids)") Collection<ResourceTag> findByResourceIds(@Param("pids") Collection<Long> pids); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java index d5569612515..e17b4d4fed4 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java @@ -4,11 +4,13 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.SqlQuery; import ca.uhn.fhir.jpa.util.TestUtil; +import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.param.ReferenceParam; import org.apache.commons.lang3.StringUtils; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.CareTeam; import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.IdType; @@ -538,6 +540,53 @@ public class FhirResourceDaoR4QueryCountTest extends BaseJpaR4Test { } + @Test + public void testSearchOnReverseInclude() { + Patient patient = new Patient(); + patient.getMeta().addTag("http://system", "value1", "display"); + patient.setId("P1"); + patient.getNameFirstRep().setFamily("FAM1"); + myPatientDao.update(patient); + + patient = new Patient(); + patient.setId("P2"); + patient.getMeta().addTag("http://system", "value1", "display"); + patient.getNameFirstRep().setFamily("FAM2"); + myPatientDao.update(patient); + + for (int i = 0; i < 3; i++) { + CareTeam ct = new CareTeam(); + ct.setId("CT1-" + i); + ct.getMeta().addTag("http://system", "value11", "display"); + ct.getSubject().setReference("Patient/P1"); + myCareTeamDao.update(ct); + + ct = new CareTeam(); + ct.setId("CT2-" + i); + ct.getMeta().addTag("http://system", "value22", "display"); + ct.getSubject().setReference("Patient/P2"); + myCareTeamDao.update(ct); + } + + SearchParameterMap map = SearchParameterMap + .newSynchronous() + .addRevInclude(CareTeam.INCLUDE_SUBJECT) + .setSort(new SortSpec(Patient.SP_NAME)); + + myCaptureQueriesListener.clear(); + IBundleProvider outcome = myPatientDao.search(map); + assertThat(toUnqualifiedVersionlessIdValues(outcome), containsInAnyOrder( + "Patient/P1", "CareTeam/CT1-0", "CareTeam/CT1-1","CareTeam/CT1-2", + "Patient/P2", "CareTeam/CT2-0", "CareTeam/CT2-1","CareTeam/CT2-2" + )); + + myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + assertEquals(4, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size()); + assertEquals(0, myCaptureQueriesListener.getInsertQueriesForCurrentThread().size()); + assertEquals(0, myCaptureQueriesListener.getUpdateQueriesForCurrentThread().size()); + assertEquals(0, myCaptureQueriesListener.getDeleteQueriesForCurrentThread().size()); + } + @Test public void testTransactionWithMultipleReferences() { Bundle input = new Bundle(); diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTag.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTag.java index d255c5d0005..7a7e338280a 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTag.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/entity/ResourceTag.java @@ -42,7 +42,7 @@ public class ResourceTag extends BaseTag { @Column(name = "PID") private Long myId; - @ManyToOne(cascade = {}) + @ManyToOne(cascade = {}, fetch = FetchType.LAZY) @JoinColumn(name = "RES_ID", referencedColumnName = "RES_ID", foreignKey = @ForeignKey(name = "FK_RESTAG_RESOURCE")) private ResourceTable myResource; diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java index 282bee12598..b099d4dc6c5 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/SearchParameterMap.java @@ -159,8 +159,9 @@ public class SearchParameterMap implements Serializable { } } - public void addRevInclude(Include theInclude) { + public SearchParameterMap addRevInclude(Include theInclude) { getRevIncludes().add(theInclude); + return this; } private void addUrlIncludeParams(StringBuilder b, String paramName, Set<Include> theList) { @@ -268,8 +269,9 @@ public class SearchParameterMap implements Serializable { return mySort; } - public void setSort(SortSpec theSort) { + public SearchParameterMap setSort(SortSpec theSort) { mySort = theSort; + return this; } /** From e65c2649277916f1fa25068ff5620b4ae4c45ab4 Mon Sep 17 00:00:00 2001 From: James Agnew <jamesagnew@gmail.com> Date: Tue, 23 Jun 2020 17:58:47 -0400 Subject: [PATCH 17/22] Support double _has expressions (#1939) * Support nested _has queries * Add changelog --- .../5_1_0/1939-double-has-expressions.yaml | 5 ++ .../predicate/PredicateBuilderReference.java | 59 ++++++++++++------- .../r4/FhirResourceDaoR4SearchNoFtTest.java | 50 ++++++++++++++++ 3 files changed, 93 insertions(+), 21 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1939-double-has-expressions.yaml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1939-double-has-expressions.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1939-double-has-expressions.yaml new file mode 100644 index 00000000000..d669397291a --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_1_0/1939-double-has-expressions.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 1939 +title: "The JPA server is now able to support _has queries where the linked search expression on the right hand side of the + _has parameter is a second _has query." diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderReference.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderReference.java index e6236013476..3703a360fa4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderReference.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderReference.java @@ -62,6 +62,7 @@ import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.param.CompositeParam; import ca.uhn.fhir.rest.param.DateParam; +import ca.uhn.fhir.rest.param.HasOrListParam; import ca.uhn.fhir.rest.param.HasParam; import ca.uhn.fhir.rest.param.NumberParam; import ca.uhn.fhir.rest.param.ParameterUtil; @@ -944,30 +945,46 @@ class PredicateBuilderReference extends BasePredicateBuilder { throw new InvalidRequestException("Invalid resource type: " + targetResourceType); } - //Ensure that the name of the search param - // (e.g. the `code` in Patient?_has:Observation:subject:code=sys|val) - // exists on the target resource type. - RuntimeSearchParam owningParameterDef = mySearchParamRegistry.getSearchParamByName(targetResourceDefinition, paramName); - if (owningParameterDef == null) { - throw new InvalidRequestException("Unknown parameter name: " + targetResourceType + ':' + parameterName); - } - - //Ensure that the name of the back-referenced search param on the target (e.g. the `subject` in Patient?_has:Observation:subject:code=sys|val) - //exists on the target resource. - owningParameterDef = mySearchParamRegistry.getSearchParamByName(targetResourceDefinition, paramReference); - if (owningParameterDef == null) { - throw new InvalidRequestException("Unknown parameter name: " + targetResourceType + ':' + paramReference); - } - - RuntimeSearchParam paramDef = mySearchParamRegistry.getSearchParamByName(targetResourceDefinition, paramName); - - IQueryParameterAnd<IQueryParameterOr<IQueryParameterType>> parsedParam = (IQueryParameterAnd<IQueryParameterOr<IQueryParameterType>>) ParameterUtil.parseQueryParams(myContext, paramDef, paramName, parameters); - ArrayList<IQueryParameterType> orValues = Lists.newArrayList(); - for (IQueryParameterOr<IQueryParameterType> next : parsedParam.getValuesAsQueryTokens()) { - orValues.addAll(next.getValuesAsQueryTokens()); + if (paramName.startsWith("_has:")) { + + ourLog.trace("Handing double _has query: {}", paramName); + + String qualifier = paramName.substring(4); + paramName = Constants.PARAM_HAS; + for (IQueryParameterType next : nextOrList) { + HasParam nextHasParam = new HasParam(); + nextHasParam.setValueAsQueryToken(myContext, Constants.PARAM_HAS, qualifier, next.getValueAsQueryToken(myContext)); + orValues.add(nextHasParam); + } + + } else { + + //Ensure that the name of the search param + // (e.g. the `code` in Patient?_has:Observation:subject:code=sys|val) + // exists on the target resource type. + RuntimeSearchParam owningParameterDef = mySearchParamRegistry.getSearchParamByName(targetResourceDefinition, paramName); + if (owningParameterDef == null) { + throw new InvalidRequestException("Unknown parameter name: " + targetResourceType + ':' + parameterName); + } + + //Ensure that the name of the back-referenced search param on the target (e.g. the `subject` in Patient?_has:Observation:subject:code=sys|val) + //exists on the target resource. + owningParameterDef = mySearchParamRegistry.getSearchParamByName(targetResourceDefinition, paramReference); + if (owningParameterDef == null) { + throw new InvalidRequestException("Unknown parameter name: " + targetResourceType + ':' + paramReference); + } + + RuntimeSearchParam paramDef = mySearchParamRegistry.getSearchParamByName(targetResourceDefinition, paramName); + IQueryParameterAnd<IQueryParameterOr<IQueryParameterType>> parsedParam = (IQueryParameterAnd<IQueryParameterOr<IQueryParameterType>>) ParameterUtil.parseQueryParams(myContext, paramDef, paramName, parameters); + + for (IQueryParameterOr<IQueryParameterType> next : parsedParam.getValuesAsQueryTokens()) { + orValues.addAll(next.getValuesAsQueryTokens()); + } + } + //Handle internal chain inside the has. if (parameterName.contains(".")) { String chainedPartOfParameter = getChainedPart(parameterName); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java index e35bc418ff5..d17a9b91613 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java @@ -876,6 +876,11 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { Observation obs = new Observation(); obs.addIdentifier().setSystem("urn:system").setValue("NOLINK"); obs.setDevice(new Reference(devId)); + IIdType obsId = myObservationDao.create(obs, mySrd).getId().toUnqualifiedVersionless(); + + DiagnosticReport dr = new DiagnosticReport(); + dr.addResult().setReference(obsId.getValue()); + dr.setStatus(DiagnosticReport.DiagnosticReportStatus.FINAL); myObservationDao.create(obs, mySrd); } @@ -896,6 +901,51 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(params)), empty()); } + @Test + public void testHasParameterDouble() { + // Matching + IIdType pid0; + { + Patient patient = new Patient(); + patient.addIdentifier().setSystem("urn:system").setValue("00"); + patient.addName().setFamily("Tester").addGiven("Joe"); + pid0 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + + Observation obs = new Observation(); + obs.addIdentifier().setSystem("urn:system").setValue("NOLINK"); + obs.setSubject(new Reference(pid0)); + IIdType obsId = myObservationDao.create(obs, mySrd).getId().toUnqualifiedVersionless(); + + DiagnosticReport dr = new DiagnosticReport(); + dr.addResult().setReference(obsId.getValue()); + dr.setStatus(DiagnosticReport.DiagnosticReportStatus.FINAL); + myDiagnosticReportDao.create(dr, mySrd); + } + + // Matching + { + Patient patient = new Patient(); + patient.addIdentifier().setSystem("urn:system").setValue("001"); + patient.addName().setFamily("Tester").addGiven("Joe"); + IIdType pid1 = myPatientDao.create(patient, mySrd).getId().toUnqualifiedVersionless(); + + Observation obs = new Observation(); + obs.addIdentifier().setSystem("urn:system").setValue("NOLINK"); + obs.setSubject(new Reference(pid1)); + myObservationDao.create(obs, mySrd).getId().toUnqualifiedVersionless(); + + } + + SearchParameterMap params = SearchParameterMap.newSynchronous(); + + // Double _has + params = new SearchParameterMap(); + params.add("_has", new HasParam("Observation", "subject", "_has:DiagnosticReport:result:status", "final")); + assertThat(toUnqualifiedVersionlessIdValues(myPatientDao.search(params)), containsInAnyOrder(pid0.getValue())); + + } + + @Test public void testHasParameterChained() { IIdType pid0; From 38a2b00663318711c4292e00d0812b2ff9fe67b9 Mon Sep 17 00:00:00 2001 From: Ken Stevens <khstevens@gmail.com> Date: Wed, 24 Jun 2020 09:12:56 -0400 Subject: [PATCH 18/22] Empi 69 and candidates (#1936) broaden empi blocking searches to support and searchparams as well as ors --- .../fhir/docs/server_jpa_empi/empi_rules.md | 53 +++++++++++--- .../jpa/empi/config/EmpiConsumerConfig.java | 6 ++ ...EmpiCandidateSearchCriteriaBuilderSvc.java | 45 ++++++++++++ .../jpa/empi/svc/EmpiCandidateSearchSvc.java | 42 ++++------- .../fhir/jpa/empi/svc/EmpiSearchParamSvc.java | 5 +- ...CandidateSearchCriteriaBuilderSvcTest.java | 69 +++++++++++++++++++ .../empi/svc/EmpiCandidateSearchSvcTest.java | 2 +- .../src/test/resources/empi/empi-rules.json | 6 +- .../empi/rules/config/EmpiRuleValidator.java | 7 +- .../json/EmpiResourceSearchParamJson.java | 21 ++++-- .../empi/rules/svc/BaseEmpiRulesR4Test.java | 4 +- .../resources/bad-rules-bad-searchparam.json | 2 +- 12 files changed, 204 insertions(+), 58 deletions(-) create mode 100644 hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java create mode 100644 hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvcTest.java diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_empi/empi_rules.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_empi/empi_rules.md index 58fb0a8fce9..d6722b08f6b 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_empi/empi_rules.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_empi/empi_rules.md @@ -11,15 +11,15 @@ Here is an example of a full HAPI EMPI rules json document: "candidateSearchParams": [ { "resourceType": "Patient", - "searchParam": "birthdate" + "searchParams": ["given", "family"] }, { "resourceType": "*", - "searchParam": "identifier" + "searchParams": ["identifier"] }, { "resourceType": "Patient", - "searchParam": "general-practitioner" + "searchParams": ["general-practitioner"] } ], "candidateFilterSearchParams": [ @@ -56,18 +56,21 @@ Here is an example of a full HAPI EMPI rules json document: Here is a description of how each section of this document is configured. -* **candidateSearchParams**: These define fields which must have at least one exact match before two resources are considered for matching. This is like a list of "pre-searches" that find potential candidates for matches, to avoid the expensive operation of running a match score calculation on all resources in the system. E.g. you may only wish to consider matching two Patients if they either share at least one identifier in common or have the same birthday. The HAPI FHIR server executes each of these searches separately and then takes the union of the results, so you can think of these as `OR` criteria that cast a wide net for potential candidates. In some EMPI systems, these "pre-searches" are called "blocking" searches (since they identify "blocks" of candidates that will be searched for matches). +### candidateSearchParams +These define fields which must have at least one exact match before two resources are considered for matching. This is like a list of "pre-searches" that find potential candidates for matches, to avoid the expensive operation of running a match score calculation on all resources in the system. E.g. you may only wish to consider matching two Patients if they either share at least one identifier in common or have the same birthday. The HAPI FHIR server executes each of these searches separately and then takes the union of the results, so you can think of these as `OR` criteria that cast a wide net for potential candidates. In some EMPI systems, these "pre-searches" are called "blocking" searches (since they identify "blocks" of candidates that will be searched for matches). + ```json [ { "resourceType" : "Patient", - "searchParam" : "birthdate" + "searchParams" : ["given", "family"] }, { "resourceType" : "Patient", "searchParam" : "identifier" } ] ``` -* **candidateFilterSearchParams** When searching for match candidates, only resources that match this filter are considered. E.g. you may wish to only search for Patients for which active=true. Another way to think of these filters is all of them are "AND"ed with each candidateSearchParam above. +### candidateFilterSearchParams +When searching for match candidates, only resources that match this filter are considered. E.g. you may wish to only search for Patients for which active=true. Another way to think of these filters is all of them are "AND"ed with each candidateSearchParam above. ```json [ { "resourceType" : "Patient", @@ -76,7 +79,35 @@ Here is a description of how each section of this document is configured. } ] ``` -* **matchFields** Once the match candidates have been found, they are then each compared to the incoming Patient resource. This comparison is made across a list of `matchField`s. Each matchField returns `true` or `false` indicating whether the candidate and the incoming Patient match on that field. There are two types of metrics: `Matcher` and `Similarity`. Matcher metrics return a `true` or `false` directly, whereas Similarity metrics return a score between 0.0 (no match) and 1.0 (exact match) and this score is translated to a `true/false` via a `matchThreshold`. E.g. if a `JARO_WINKLER` matchField is configured with a `matchThreshold` of 0.8 then that matchField will return `true` if the `JARO_WINKLER` similarity evaluates to a score >= 8.0. +For example, if the incoming patient looked like this: + +```json +{ + "resourceType": "Patient", + "id": "example", + "identifier": [{ + "system": "urn:oid:1.2.36.146.595.217.0.1", + "value": "12345" + }], + "name": [ + { + "family": "Chalmers", + "given": [ + "Peter", + "James" + ] + } +} +``` + +then the above `candidateSearchParams` and `candidateFilterSearchParams` would result in the following two consecutive searches for candidates: +* `Patient?given=Peter,James&family=Chalmers&active=true` +* `Patient?identifier=urn:oid:1.2.36.146.595.217.0.1|12345&active=true` + + +### matchFields + +Once the match candidates have been found, they are then each compared to the incoming Patient resource. This comparison is made across a list of `matchField`s. Each matchField returns `true` or `false` indicating whether the candidate and the incoming Patient match on that field. There are two types of metrics: `Matcher` and `Similarity`. Matcher metrics return a `true` or `false` directly, whereas Similarity metrics return a score between 0.0 (no match) and 1.0 (exact match) and this score is translated to a `true/false` via a `matchThreshold`. E.g. if a `JARO_WINKLER` matchField is configured with a `matchThreshold` of 0.8 then that matchField will return `true` if the `JARO_WINKLER` similarity evaluates to a score >= 8.0. By default, all matchFields have `exact=false` which means that they will have all diacritical marks removed and converted to upper case before matching. `exact=true` can be added to any matchField to compare the strings as they are originally capitalized and accented. @@ -250,7 +281,9 @@ The following metrics are currently supported: </tbody> </table> -* **matchResultMap** converts combinations of successful matchFields into an EMPI Match Result for overall matching of a given pair of resources. MATCH results are evaluated take precedence over POSSIBLE_MATCH results. +### matchResultMap + +These entries convert combinations of successful matchFields into an EMPI Match Result for overall matching of a given pair of resources. MATCH results are evaluated take precedence over POSSIBLE_MATCH results. ```json { @@ -261,4 +294,6 @@ The following metrics are currently supported: } ``` -* **eidSystem**: The external EID system that the HAPI EMPI system should expect to see on incoming Patient resources. Must be a valid URI. See [EMPI EID](/hapi-fhir/docs/server_jpa_empi/empi_eid.html) for details on how EIDs are managed by HAPI EMPI. +### eidSystem + +The external EID system that the HAPI EMPI system should expect to see on incoming Patient resources. Must be a valid URI. See [EMPI EID](/hapi-fhir/docs/server_jpa_empi/empi_eid.html) for details on how EIDs are managed by HAPI EMPI. diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/config/EmpiConsumerConfig.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/config/EmpiConsumerConfig.java index e62fd13ae3c..6ae308bb772 100644 --- a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/config/EmpiConsumerConfig.java +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/config/EmpiConsumerConfig.java @@ -37,6 +37,7 @@ import ca.uhn.fhir.jpa.empi.broker.EmpiMessageHandler; import ca.uhn.fhir.jpa.empi.broker.EmpiQueueConsumerLoader; import ca.uhn.fhir.jpa.empi.interceptor.EmpiStorageInterceptor; import ca.uhn.fhir.jpa.empi.interceptor.IEmpiStorageInterceptor; +import ca.uhn.fhir.jpa.empi.svc.EmpiCandidateSearchCriteriaBuilderSvc; import ca.uhn.fhir.jpa.empi.svc.EmpiCandidateSearchSvc; import ca.uhn.fhir.jpa.empi.svc.EmpiEidUpdateService; import ca.uhn.fhir.jpa.empi.svc.EmpiLinkQuerySvcImpl; @@ -157,6 +158,11 @@ public class EmpiConsumerConfig { return new EmpiCandidateSearchSvc(); } + @Bean + EmpiCandidateSearchCriteriaBuilderSvc empiCriteriaBuilderSvc() { + return new EmpiCandidateSearchCriteriaBuilderSvc(); + } + @Bean EmpiResourceMatcherSvc empiResourceComparatorSvc(FhirContext theFhirContext, IEmpiSettings theEmpiConfig) { return new EmpiResourceMatcherSvc(theFhirContext, theEmpiConfig); diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java new file mode 100644 index 00000000000..50289f0b69e --- /dev/null +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java @@ -0,0 +1,45 @@ +package ca.uhn.fhir.jpa.empi.svc; + +import ca.uhn.fhir.empi.rules.json.EmpiResourceSearchParamJson; +import org.hl7.fhir.instance.model.api.IAnyResource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +@Service +public class EmpiCandidateSearchCriteriaBuilderSvc { + @Autowired + private EmpiSearchParamSvc myEmpiSearchParamSvc; + + /* + * Given a list of criteria upon which to block, a resource search parameter, and a list of values for that given search parameter, + * build a query url. e.g. + * + * Patient?active=true&name.given=Gary,Grant&name.family=Graham + */ + @Nonnull + public Optional<String> buildResourceQueryString(String theResourceType, IAnyResource theResource, List<String> theFilterCriteria, EmpiResourceSearchParamJson resourceSearchParam) { + List<String> criteria = new ArrayList<>(); + + resourceSearchParam.iterator().forEachRemaining(searchParam -> { + //to compare it to all known PERSON objects, using the overlapping search parameters that they have. + List<String> valuesFromResourceForSearchParam = myEmpiSearchParamSvc.getValueFromResourceForSearchParam(theResource, searchParam); + if (!valuesFromResourceForSearchParam.isEmpty()) { + criteria.add(buildResourceMatchQuery(searchParam, valuesFromResourceForSearchParam)); + } + }); + if (criteria.isEmpty()) { + return Optional.empty(); + } + criteria.addAll(theFilterCriteria); + return Optional.of(theResourceType + "?" + String.join("&", criteria)); + } + + private String buildResourceMatchQuery(String theSearchParamName, List<String> theResourceValues) { + return theSearchParamName + "=" + String.join(",", theResourceValues); + } +} diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvc.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvc.java index edb4e58ae1d..88d420df19f 100644 --- a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvc.java +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvc.java @@ -35,13 +35,12 @@ import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import javax.annotation.Nonnull; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import static ca.uhn.fhir.empi.api.EmpiConstants.ALL_RESOURCE_SEARCH_PARAM_TYPE; @@ -58,6 +57,11 @@ public class EmpiCandidateSearchSvc { private DaoRegistry myDaoRegistry; @Autowired private IdHelperService myIdHelperService; + @Autowired + private EmpiCandidateSearchCriteriaBuilderSvc myEmpiCandidateSearchCriteriaBuilderSvc; + + public EmpiCandidateSearchSvc() { + } /** * Given a target resource, search for all resources that are considered an EMPI match based on defined EMPI rules. @@ -81,13 +85,7 @@ public class EmpiCandidateSearchSvc { continue; } - //to compare it to all known PERSON objects, using the overlapping search parameters that they have. - List<String> valuesFromResourceForSearchParam = myEmpiSearchParamSvc.getValueFromResourceForSearchParam(theResource, resourceSearchParam); - if (valuesFromResourceForSearchParam.isEmpty()) { - continue; - } - - searchForIdsAndAddToMap(theResourceType, matchedPidsToResources, filterCriteria, resourceSearchParam, valuesFromResourceForSearchParam); + searchForIdsAndAddToMap(theResourceType, theResource, matchedPidsToResources, filterCriteria, resourceSearchParam); } //Obviously we don't want to consider the freshly added resource as a potential candidate. //Sometimes, we are running this function on a resource that has not yet been persisted, @@ -111,9 +109,13 @@ public class EmpiCandidateSearchSvc { * 4. Store all results in `theMatchedPidsToResources` */ @SuppressWarnings("rawtypes") - private void searchForIdsAndAddToMap(String theResourceType, Map<Long, IAnyResource> theMatchedPidsToResources, List<String> theFilterCriteria, EmpiResourceSearchParamJson resourceSearchParam, List<String> theValuesFromResourceForSearchParam) { + private void searchForIdsAndAddToMap(String theResourceType, IAnyResource theResource, Map<Long, IAnyResource> theMatchedPidsToResources, List<String> theFilterCriteria, EmpiResourceSearchParamJson resourceSearchParam) { //1. - String resourceCriteria = buildResourceQueryString(theResourceType, theFilterCriteria, resourceSearchParam, theValuesFromResourceForSearchParam); + Optional<String> oResourceCriteria = myEmpiCandidateSearchCriteriaBuilderSvc.buildResourceQueryString(theResourceType, theResource, theFilterCriteria, resourceSearchParam); + if (!oResourceCriteria.isPresent()) { + return; + } + String resourceCriteria = oResourceCriteria.get(); ourLog.debug("Searching for {} candidates with {}", theResourceType, resourceCriteria); //2. @@ -139,24 +141,6 @@ public class EmpiCandidateSearchSvc { } } - /* - * Given a list of criteria upon which to block, a resource search parameter, and a list of values for that given search parameter, - * build a query url. e.g. - * - * Patient?active=true&name.given=Gary,Grant - */ - @Nonnull - private String buildResourceQueryString(String theResourceType, List<String> theFilterCriteria, EmpiResourceSearchParamJson resourceSearchParam, List<String> theValuesFromResourceForSearchParam) { - List<String> criteria = new ArrayList<>(theFilterCriteria); - criteria.add(buildResourceMatchQuery(resourceSearchParam.getSearchParam(), theValuesFromResourceForSearchParam)); - - return theResourceType + "?" + String.join("&", criteria); - } - - private String buildResourceMatchQuery(String theSearchParamName, List<String> theResourceValues) { - return theSearchParamName + "=" + String.join(",", theResourceValues); - } - private List<String> buildFilterQuery(List<EmpiFilterSearchParamJson> theFilterSearchParams, String theResourceType) { return Collections.unmodifiableList(theFilterSearchParams.stream() .filter(spFilterJson -> paramIsOnCorrectType(theResourceType, spFilterJson)) diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiSearchParamSvc.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiSearchParamSvc.java index 42bc5d0b7c3..6e7c0cddd4e 100644 --- a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiSearchParamSvc.java +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiSearchParamSvc.java @@ -23,7 +23,6 @@ package ca.uhn.fhir.jpa.empi.svc; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.context.RuntimeSearchParam; -import ca.uhn.fhir.empi.rules.json.EmpiResourceSearchParamJson; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.extractor.SearchParamExtractorService; @@ -51,9 +50,9 @@ public class EmpiSearchParamSvc implements ISearchParamRetriever { return myMatchUrlService.translateMatchUrl(theResourceCriteria, resourceDef); } - public List<String> getValueFromResourceForSearchParam(IBaseResource theResource, EmpiResourceSearchParamJson theFilterSearchParam) { + public List<String> getValueFromResourceForSearchParam(IBaseResource theResource, String theSearchParam) { String resourceType = myFhirContext.getResourceType(theResource); - RuntimeSearchParam activeSearchParam = mySearchParamRegistry.getActiveSearchParam(resourceType, theFilterSearchParam.getSearchParam()); + RuntimeSearchParam activeSearchParam = mySearchParamRegistry.getActiveSearchParam(resourceType, theSearchParam); return mySearchParamExtractorService.extractParamValuesAsStrings(activeSearchParam, theResource); } diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvcTest.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvcTest.java new file mode 100644 index 00000000000..93c34614a57 --- /dev/null +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvcTest.java @@ -0,0 +1,69 @@ +package ca.uhn.fhir.jpa.empi.svc; + +import ca.uhn.fhir.empi.rules.json.EmpiResourceSearchParamJson; +import ca.uhn.fhir.jpa.empi.BaseEmpiR4Test; +import org.hl7.fhir.r4.model.HumanName; +import org.hl7.fhir.r4.model.Patient; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Collections; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class EmpiCandidateSearchCriteriaBuilderSvcTest extends BaseEmpiR4Test { + @Autowired + EmpiCandidateSearchCriteriaBuilderSvc myEmpiCandidateSearchCriteriaBuilderSvc; + + @Test + public void testEmptyCase() { + Patient patient = new Patient(); + EmpiResourceSearchParamJson searchParamJson = new EmpiResourceSearchParamJson(); + searchParamJson.addSearchParam("family"); + Optional<String> result = myEmpiCandidateSearchCriteriaBuilderSvc.buildResourceQueryString("Patient", patient, Collections.emptyList(), searchParamJson); + assertFalse(result.isPresent()); + } + + @Test + public void testSimpleCase() { + Patient patient = new Patient(); + patient.addName().setFamily("Fernandez"); + EmpiResourceSearchParamJson searchParamJson = new EmpiResourceSearchParamJson(); + searchParamJson.addSearchParam("family"); + Optional<String> result = myEmpiCandidateSearchCriteriaBuilderSvc.buildResourceQueryString("Patient", patient, Collections.emptyList(), searchParamJson); + assertTrue(result.isPresent()); + assertEquals("Patient?family=Fernandez", result.get()); + } + + @Test + public void testComplexCase() { + Patient patient = new Patient(); + HumanName humanName = patient.addName(); + humanName.addGiven("Jose"); + humanName.addGiven("Martin"); + humanName.setFamily("Fernandez"); + EmpiResourceSearchParamJson searchParamJson = new EmpiResourceSearchParamJson(); + searchParamJson.addSearchParam("given"); + searchParamJson.addSearchParam("family"); + Optional<String> result = myEmpiCandidateSearchCriteriaBuilderSvc.buildResourceQueryString("Patient", patient, Collections.emptyList(), searchParamJson); + assertTrue(result.isPresent()); + assertThat(result.get(), anyOf(equalTo("Patient?given=Jose,Martin&family=Fernandez"), equalTo("Patient?given=Martin,Jose&family=Fernandez"))); + } + + @Test + public void testIdentifier() { + Patient patient = new Patient(); + patient.addIdentifier().setSystem("urn:oid:1.2.36.146.595.217.0.1").setValue("12345"); + EmpiResourceSearchParamJson searchParamJson = new EmpiResourceSearchParamJson(); + searchParamJson.addSearchParam("identifier"); + Optional<String> result = myEmpiCandidateSearchCriteriaBuilderSvc.buildResourceQueryString("Patient", patient, Collections.emptyList(), searchParamJson); + assertTrue(result.isPresent()); + assertEquals(result.get(), "Patient?identifier=urn:oid:1.2.36.146.595.217.0.1|12345"); + } +} diff --git a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvcTest.java b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvcTest.java index edf6b38fb75..b916bbe9300 100644 --- a/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvcTest.java +++ b/hapi-fhir-jpaserver-empi/src/test/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchSvcTest.java @@ -25,7 +25,7 @@ public class EmpiCandidateSearchSvcTest extends BaseEmpiR4Test { public void testFindCandidates() { Patient jane = buildJanePatient(); jane.setActive(true); - Patient createdJane = createPatient(jane); + createPatient(jane); Patient newJane = buildJanePatient(); Collection<IAnyResource> result = myEmpiCandidateSearchSvc.findCandidates("Patient", newJane); diff --git a/hapi-fhir-jpaserver-empi/src/test/resources/empi/empi-rules.json b/hapi-fhir-jpaserver-empi/src/test/resources/empi/empi-rules.json index b45f775d0f1..7c1d675ad0c 100644 --- a/hapi-fhir-jpaserver-empi/src/test/resources/empi/empi-rules.json +++ b/hapi-fhir-jpaserver-empi/src/test/resources/empi/empi-rules.json @@ -2,15 +2,15 @@ "candidateSearchParams": [ { "resourceType": "Patient", - "searchParam": "birthdate" + "searchParams": ["birthdate"] }, { "resourceType": "*", - "searchParam": "identifier" + "searchParams": ["identifier"] }, { "resourceType": "Patient", - "searchParam": "general-practitioner" + "searchParams": ["general-practitioner"] } ], "candidateFilterSearchParams": [ diff --git a/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/config/EmpiRuleValidator.java b/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/config/EmpiRuleValidator.java index 2714d10abed..1829bd50cbe 100644 --- a/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/config/EmpiRuleValidator.java +++ b/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/config/EmpiRuleValidator.java @@ -67,8 +67,9 @@ public class EmpiRuleValidator { } private void validateSearchParams(EmpiRulesJson theEmpiRulesJson) { - for (EmpiResourceSearchParamJson searchParam : theEmpiRulesJson.getCandidateSearchParams()) { - validateSearchParam("candidateSearchParams", searchParam.getResourceType(), searchParam.getSearchParam()); + for (EmpiResourceSearchParamJson searchParams : theEmpiRulesJson.getCandidateSearchParams()) { + searchParams.iterator().forEachRemaining( + searchParam -> validateSearchParam("candidateSearchParams", searchParams.getResourceType(), searchParam)); } for (EmpiFilterSearchParamJson filter : theEmpiRulesJson.getCandidateFilterSearchParams()) { validateSearchParam("candidateFilterSearchParams", filter.getResourceType(), filter.getSearchParam()); @@ -129,7 +130,7 @@ public class EmpiRuleValidator { private void validatePatientPath(EmpiFieldMatchJson theFieldMatch) { try { myTerser.getDefinition(myPatientClass, "Patient." + theFieldMatch.getResourcePath()); - } catch (DataFormatException|ConfigurationException e) { + } catch (DataFormatException | ConfigurationException e) { throw new ConfigurationException("MatchField " + theFieldMatch.getName() + " resourceType " + diff --git a/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/json/EmpiResourceSearchParamJson.java b/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/json/EmpiResourceSearchParamJson.java index 34b81575568..9cc3e78fec8 100644 --- a/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/json/EmpiResourceSearchParamJson.java +++ b/hapi-fhir-server-empi/src/main/java/ca/uhn/fhir/empi/rules/json/EmpiResourceSearchParamJson.java @@ -23,14 +23,18 @@ package ca.uhn.fhir.empi.rules.json; import ca.uhn.fhir.model.api.IModelJson; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + /** * */ -public class EmpiResourceSearchParamJson implements IModelJson { +public class EmpiResourceSearchParamJson implements IModelJson, Iterable<String> { @JsonProperty(value = "resourceType", required = true) String myResourceType; - @JsonProperty(value = "searchParam", required = true) - String mySearchParam; + @JsonProperty(value = "searchParams", required = true) + List<String> mySearchParams; public String getResourceType() { return myResourceType; @@ -41,12 +45,15 @@ public class EmpiResourceSearchParamJson implements IModelJson { return this; } - public String getSearchParam() { - return mySearchParam; + public Iterator<String> iterator() { + return mySearchParams.iterator(); } - public EmpiResourceSearchParamJson setSearchParam(String theSearchParam) { - mySearchParam = theSearchParam; + public EmpiResourceSearchParamJson addSearchParam(String theSearchParam) { + if (mySearchParams == null) { + mySearchParams = new ArrayList<>(); + } + mySearchParams.add(theSearchParam); return this; } } diff --git a/hapi-fhir-server-empi/src/test/java/ca/uhn/fhir/empi/rules/svc/BaseEmpiRulesR4Test.java b/hapi-fhir-server-empi/src/test/java/ca/uhn/fhir/empi/rules/svc/BaseEmpiRulesR4Test.java index f9c02d9ac89..08aeb421b4c 100644 --- a/hapi-fhir-server-empi/src/test/java/ca/uhn/fhir/empi/rules/svc/BaseEmpiRulesR4Test.java +++ b/hapi-fhir-server-empi/src/test/java/ca/uhn/fhir/empi/rules/svc/BaseEmpiRulesR4Test.java @@ -37,10 +37,10 @@ public abstract class BaseEmpiRulesR4Test extends BaseR4Test { EmpiResourceSearchParamJson patientBirthdayBlocking = new EmpiResourceSearchParamJson() .setResourceType("Patient") - .setSearchParam(Patient.SP_BIRTHDATE); + .addSearchParam(Patient.SP_BIRTHDATE); EmpiResourceSearchParamJson patientIdentifierBlocking = new EmpiResourceSearchParamJson() .setResourceType("Patient") - .setSearchParam(Patient.SP_IDENTIFIER); + .addSearchParam(Patient.SP_IDENTIFIER); EmpiFieldMatchJson lastNameMatchField = new EmpiFieldMatchJson() diff --git a/hapi-fhir-server-empi/src/test/resources/bad-rules-bad-searchparam.json b/hapi-fhir-server-empi/src/test/resources/bad-rules-bad-searchparam.json index b9f1274f771..888c704517b 100644 --- a/hapi-fhir-server-empi/src/test/resources/bad-rules-bad-searchparam.json +++ b/hapi-fhir-server-empi/src/test/resources/bad-rules-bad-searchparam.json @@ -1,7 +1,7 @@ { "candidateSearchParams" : [{ "resourceType" : "Patient", - "searchParam" : "foo" + "searchParams" : ["foo"] }], "candidateFilterSearchParams" : [], "matchFields" : [], From 5e3e7c46a88446b83291a1f8e6849e94621ca19b Mon Sep 17 00:00:00 2001 From: ianmarshall <ian@simpatico.ai> Date: Wed, 24 Jun 2020 15:48:06 -0400 Subject: [PATCH 19/22] Changes to support valueset expansion with Elasticsearch. --- ...asticsearchHibernatePropertiesBuilder.java | 4 + .../elastic/ElasticsearchMappingProvider.java | 2 +- ...sourceDaoR4TerminologyElasticsearchIT.java | 157 +++++++++++++ .../ValueSetExpansionR4ElasticsearchIT.java | 216 ++++++++++++++++++ 4 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyElasticsearchIT.java create mode 100644 hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4ElasticsearchIT.java diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchHibernatePropertiesBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchHibernatePropertiesBuilder.java index 3435d733f61..adf4dc89937 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchHibernatePropertiesBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchHibernatePropertiesBuilder.java @@ -75,6 +75,10 @@ public class ElasticsearchHibernatePropertiesBuilder { theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.INDEX_MANAGEMENT_WAIT_TIMEOUT, Long.toString(myIndexManagementWaitTimeoutMillis)); theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.REQUIRED_INDEX_STATUS, myRequiredIndexStatus.getElasticsearchString()); + // Need the mapping to be dynamic because of terminology indexes. + theProperties.put("hibernate.search.default.elasticsearch.dynamic_mapping", "true"); + + // Only for unit tests theProperties.put("hibernate.search.default." + ElasticsearchEnvironment.REFRESH_AFTER_WRITE, Boolean.toString(myDebugRefreshAfterWrite)); theProperties.put("hibernate.search." + ElasticsearchEnvironment.LOG_JSON_PRETTY_PRINTING, Boolean.toString(myDebugPrettyPrintJsonLog)); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchMappingProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchMappingProvider.java index 04407e5918b..f3df5807c0f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchMappingProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/elastic/ElasticsearchMappingProvider.java @@ -51,7 +51,7 @@ public class ElasticsearchMappingProvider implements ElasticsearchAnalysisDefini builder.analyzer("standardAnalyzer").withTokenizer("standard").withTokenFilters("lowercase"); - builder.analyzer("exactAnalyzer").withTokenizer("standard"); + builder.analyzer("exactAnalyzer").withTokenizer("keyword"); builder.analyzer("conceptParentPidsAnalyzer").withTokenizer("whitespace"); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyElasticsearchIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyElasticsearchIT.java new file mode 100644 index 00000000000..9f85e3f56c6 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4TerminologyElasticsearchIT.java @@ -0,0 +1,157 @@ +package ca.uhn.fhir.jpa.dao.r4; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet; +import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; +import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; +import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticSearch; +import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.dao.data.IResourceTableDao; +import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; +import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; +import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; +import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.util.TestUtil; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.CodeSystem.CodeSystemContentMode; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.ValueSet; +import org.hl7.fhir.r4.model.ValueSet.ConceptReferenceComponent; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.Assert.assertThat; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = {TestR4ConfigWithElasticSearch.class}) +public class FhirResourceDaoR4TerminologyElasticsearchIT extends BaseJpaTest { + + public static final String URL_MY_CODE_SYSTEM = "http://example.com/my_code_system"; + public static final String URL_MY_VALUE_SET = "http://example.com/my_value_set"; + private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FhirResourceDaoR4TerminologyElasticsearchIT.class); + + @Autowired + protected DaoConfig myDaoConfig; + @Autowired + @Qualifier("myCodeSystemDaoR4") + protected IFhirResourceDaoCodeSystem<CodeSystem, Coding, CodeableConcept> myCodeSystemDao; + @Autowired + protected IResourceTableDao myResourceTableDao; + @Autowired + protected ITermCodeSystemStorageSvc myTermCodeSystemStorageSvc; + @Autowired + @Qualifier("myValueSetDaoR4") + protected IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao; + @Autowired + FhirContext myFhirContext; + @Autowired + PlatformTransactionManager myTxManager; + + @Autowired + private IFhirSystemDao mySystemDao; + @Autowired + private IResourceReindexingSvc myResourceReindexingSvc; + @Autowired + private ISearchCoordinatorSvc mySearchCoordinatorSvc; + @Autowired + private ISearchParamRegistry mySearchParamRegistry; + @Autowired + private IBulkDataExportSvc myBulkDataExportSvc; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + protected ServletRequestDetails mySrd; + + @Test + public void testExpandWithIncludeContainingDashesInInclude() { + CodeSystem codeSystem = new CodeSystem(); + codeSystem.setUrl(URL_MY_CODE_SYSTEM); + codeSystem.setContent(CodeSystemContentMode.NOTPRESENT); + IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); + + ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalStateException::new); + + TermCodeSystemVersion cs = new TermCodeSystemVersion(); + cs.setResource(table); + + TermConcept concept; + concept = new TermConcept(cs, "LA1111-2"); + cs.getConcepts().add(concept); + concept = new TermConcept(cs, "LA2222-2"); + cs.getConcepts().add(concept); + concept = new TermConcept(cs, "LA3333-2"); + cs.getConcepts().add(concept); + concept = new TermConcept(cs, "LA1122-2"); + cs.getConcepts().add(concept); + concept = new TermConcept(cs, "LA1133-2"); + cs.getConcepts().add(concept); + concept = new TermConcept(cs, "LA4444-2"); + cs.getConcepts().add(concept); + concept = new TermConcept(cs, "LA9999-7"); + cs.getConcepts().add(concept); + + myTermCodeSystemStorageSvc.storeNewCodeSystemVersion(new ResourcePersistentId(table.getId()), URL_MY_CODE_SYSTEM, "SYSTEM NAME", "SYSTEM VERSION" , cs, table); + + ValueSet valueSet = new ValueSet(); + valueSet.setUrl(URL_MY_VALUE_SET); + valueSet.getCompose() + .addInclude() + .setSystem(codeSystem.getUrl()) + .addConcept(new ConceptReferenceComponent().setCode("LA2222-2")) + .addConcept(new ConceptReferenceComponent().setCode("LA1122-2")); + IIdType vsId = myValueSetDao.create(valueSet, mySrd).getId().toUnqualifiedVersionless(); + + ValueSet expansion = myValueSetDao.expand(vsId, null, null); + Set<String> codes = expansion + .getExpansion() + .getContains() + .stream() + .map(ValueSet.ValueSetExpansionContainsComponent::getCode) + .collect(Collectors.toSet()); + ourLog.info("Codes: {}", codes); + assertThat(codes, containsInAnyOrder("LA2222-2", "LA1122-2")); + } + + + @Override + protected FhirContext getContext() { + return myFhirContext; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myTxManager; + } + + @After + public void afterPurgeDatabase() { + purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc); + } + + @AfterClass + public static void afterClassClearContext() { + TestUtil.clearAllStaticFieldsForUnitTest(); + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4ElasticsearchIT.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4ElasticsearchIT.java new file mode 100644 index 00000000000..c2535d520c0 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/ValueSetExpansionR4ElasticsearchIT.java @@ -0,0 +1,216 @@ +package ca.uhn.fhir.jpa.term; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet; +import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; +import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; +import ca.uhn.fhir.jpa.bulk.IBulkDataExportSvc; +import ca.uhn.fhir.jpa.config.TestR4ConfigWithElasticSearch; +import ca.uhn.fhir.jpa.dao.BaseJpaTest; +import ca.uhn.fhir.jpa.dao.data.IResourceTableDao; +import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; +import ca.uhn.fhir.jpa.entity.TermConcept; +import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; +import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; +import ca.uhn.fhir.jpa.searchparam.registry.ISearchParamRegistry; +import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc; +import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; +import ca.uhn.fhir.jpa.term.api.ITermReadSvcR4; +import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.CodeSystem; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.ValueSet; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = {TestR4ConfigWithElasticSearch.class}) +public class ValueSetExpansionR4ElasticsearchIT extends BaseJpaTest { + + @Autowired + protected DaoConfig myDaoConfig; + @Autowired + @Qualifier("myCodeSystemDaoR4") + protected IFhirResourceDaoCodeSystem<CodeSystem, Coding, CodeableConcept> myCodeSystemDao; + @Autowired + protected IResourceTableDao myResourceTableDao; + @Autowired + protected ITermCodeSystemStorageSvc myTermCodeSystemStorageSvc; + @Autowired + @Qualifier("myValueSetDaoR4") + protected IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao; + @Autowired + protected ITermReadSvcR4 myTermSvc; + @Autowired + protected ITermDeferredStorageSvc myTerminologyDeferredStorageSvc; + + @Autowired + FhirContext myFhirContext; + @Autowired + PlatformTransactionManager myTxManager; + + @Autowired + private IFhirSystemDao mySystemDao; + @Autowired + private IResourceReindexingSvc myResourceReindexingSvc; + @Autowired + private ISearchCoordinatorSvc mySearchCoordinatorSvc; + @Autowired + private ISearchParamRegistry mySearchParamRegistry; + @Autowired + private IBulkDataExportSvc myBulkDataExportSvc; + + @Mock + private IValueSetConceptAccumulator myValueSetCodeAccumulator; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + protected ServletRequestDetails mySrd; + + protected static final String CS_URL = "http://example.com/my_code_system"; + + @After + public void after() { + myDaoConfig.setMaximumExpansionSize(DaoConfig.DEFAULT_MAX_EXPANSION_SIZE); + } + + @After + public void afterPurgeDatabase() { + purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc); + } + + void createCodeSystem() { + CodeSystem codeSystem = new CodeSystem(); + codeSystem.setUrl(CS_URL); + codeSystem.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT); + codeSystem.setName("SYSTEM NAME"); + codeSystem.setVersion("SYSTEM VERSION"); + IIdType id = myCodeSystemDao.create(codeSystem, mySrd).getId().toUnqualified(); + + ResourceTable table = myResourceTableDao.findById(id.getIdPartAsLong()).orElseThrow(IllegalArgumentException::new); + + TermCodeSystemVersion cs = new TermCodeSystemVersion(); + cs.setResource(table); + + TermConcept parent; + parent = new TermConcept(cs, "ParentWithNoChildrenA"); + cs.getConcepts().add(parent); + parent = new TermConcept(cs, "ParentWithNoChildrenB"); + cs.getConcepts().add(parent); + parent = new TermConcept(cs, "ParentWithNoChildrenC"); + cs.getConcepts().add(parent); + + TermConcept parentA = new TermConcept(cs, "ParentA"); + cs.getConcepts().add(parentA); + + TermConcept childAA = new TermConcept(cs, "childAA"); + parentA.addChild(childAA, TermConceptParentChildLink.RelationshipTypeEnum.ISA); + + TermConcept childAAA = new TermConcept(cs, "childAAA"); + childAAA.addPropertyString("propA", "valueAAA"); + childAAA.addPropertyString("propB", "foo"); + childAA.addChild(childAAA, TermConceptParentChildLink.RelationshipTypeEnum.ISA); + + TermConcept childAAB = new TermConcept(cs, "childAAB"); + childAAB.addPropertyString("propA", "valueAAB"); + childAAB.addPropertyString("propB", "foo"); + childAAB.addDesignation() + .setUseSystem("D1S") + .setUseCode("D1C") + .setUseDisplay("D1D") + .setValue("D1V"); + childAA.addChild(childAAB, TermConceptParentChildLink.RelationshipTypeEnum.ISA); + + TermConcept childAB = new TermConcept(cs, "childAB"); + parentA.addChild(childAB, TermConceptParentChildLink.RelationshipTypeEnum.ISA); + + TermConcept parentB = new TermConcept(cs, "ParentB"); + cs.getConcepts().add(parentB); + + myTermCodeSystemStorageSvc.storeNewCodeSystemVersion(new ResourcePersistentId(table.getId()), CS_URL, "SYSTEM NAME", "SYSTEM VERSION", cs, table); + + } + + @Test + public void testExpandValueSetInMemoryRespectsMaxSize() { + createCodeSystem(); + + // Add lots more codes + CustomTerminologySet additions = new CustomTerminologySet(); + for (int i = 0; i < 100; i++) { + additions.addRootConcept("CODE" + i, "Display " + i); + } + myTermCodeSystemStorageSvc.applyDeltaCodeSystemsAdd(CS_URL, additions); + + + // Codes available exceeds the max + myDaoConfig.setMaximumExpansionSize(50); + ValueSet vs = new ValueSet(); + ValueSet.ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(CS_URL); + try { + myTermSvc.expandValueSet(null, vs); + fail(); + } catch (InternalErrorException e) { + assertEquals("Expansion of ValueSet produced too many codes (maximum 50) - Operation aborted!", e.getMessage()); + } + + // Increase the max so it won't exceed + myDaoConfig.setMaximumExpansionSize(150); + vs = new ValueSet(); + include = vs.getCompose().addInclude(); + include.setSystem(CS_URL); + ValueSet outcome = myTermSvc.expandValueSet(null, vs); + assertEquals(109, outcome.getExpansion().getContains().size()); + + } + + @Test + public void testExpandValueSetWithValueSetCodeAccumulator() { + createCodeSystem(); + + when(myValueSetCodeAccumulator.getCapacityRemaining()).thenReturn(100); + + ValueSet vs = new ValueSet(); + ValueSet.ConceptSetComponent include = vs.getCompose().addInclude(); + include.setSystem(CS_URL); + + myTermSvc.expandValueSet(null, vs, myValueSetCodeAccumulator); + verify(myValueSetCodeAccumulator, times(9)).includeConceptWithDesignations(anyString(), anyString(), nullable(String.class), anyCollection()); + } + + + @Override + protected FhirContext getContext() { + return myFhirContext; + } + + @Override + protected PlatformTransactionManager getTxManager() { + return myTxManager; + } +} From 053094f2a697c3766062e1aa4aee002c1ba47f00 Mon Sep 17 00:00:00 2001 From: ianmarshall <ian@simpatico.ai> Date: Wed, 24 Jun 2020 17:42:46 -0400 Subject: [PATCH 20/22] Apply recent fixes for MySQL migration tasks to MariaDB as well. --- .../java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java | 2 +- .../java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java | 5 +++-- .../ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java | 2 +- .../java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java | 4 +--- .../ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java | 4 +--- .../ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java | 2 +- .../jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java | 2 +- 7 files changed, 9 insertions(+), 12 deletions(-) diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java index bf76353ce9a..fe9e0f0f5a8 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/AddColumnTask.java @@ -54,11 +54,11 @@ public class AddColumnTask extends BaseTableColumnTypeTask { String sql; switch (getDriverType()) { case MYSQL_5_7: + case MARIADB_10_1: // Quote the column name as "SYSTEM" is a reserved word in MySQL sql = "alter table " + getTableName() + " add column `" + getColumnName() + "` " + typeStatement; break; case DERBY_EMBEDDED: - case MARIADB_10_1: case POSTGRES_9_4: sql = "alter table " + getTableName() + " add column " + getColumnName() + " " + typeStatement; break; diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java index 8b7ff289526..9b1d7816e31 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropColumnTask.java @@ -26,6 +26,7 @@ import org.intellij.lang.annotations.Language; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Driver; import java.sql.SQLException; import java.util.List; import java.util.Set; @@ -52,8 +53,8 @@ public class DropColumnTask extends BaseTableColumnTask { return; } - if(getDriverType().equals(DriverTypeEnum.MYSQL_5_7)) { - // Some DBs such as MYSQL require that foreign keys depending on the column be dropped before the column itself is dropped. + if(getDriverType().equals(DriverTypeEnum.MYSQL_5_7) || getDriverType().equals(DriverTypeEnum.MARIADB_10_1)) { + // Some DBs such as MYSQL and Maria DB require that foreign keys depending on the column be dropped before the column itself is dropped. logInfo(ourLog, "Dropping any foreign keys on table {} depending on column {}", getTableName(), getColumnName()); Set<String> foreignKeys = JdbcUtils.getForeignKeysForColumn(getConnectionProperties(), getColumnName(), getTableName()); if(foreignKeys != null) { diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java index 088682ffc1d..ac0f156bff5 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropForeignKeyTask.java @@ -101,10 +101,10 @@ public class DropForeignKeyTask extends BaseTableTask { List<String> sqls = new ArrayList<>(); switch (theDriverType) { case MYSQL_5_7: + case MARIADB_10_1: // Lousy MYQL.... sqls.add("alter table " + theTableName + " drop foreign key " + theConstraintName); break; - case MARIADB_10_1: case POSTGRES_9_4: case DERBY_EMBEDDED: case H2_EMBEDDED: diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java index cbac30507c0..9d5b3171f3b 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/DropIndexTask.java @@ -107,12 +107,10 @@ public class DropIndexTask extends BaseTableTask { // Drop constraint switch (theDriverType) { case MYSQL_5_7: + case MARIADB_10_1: // Need to quote the index name as the word "PRIMARY" is reserved in MySQL sql.add("alter table " + theTableName + " drop index `" + theIndexName + "`"); break; - case MARIADB_10_1: - sql.add("alter table " + theTableName + " drop index " + theIndexName); - break; case H2_EMBEDDED: sql.add("drop index " + theIndexName); break; diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java index 7e40827ffa1..49abd9b5a83 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTask.java @@ -139,10 +139,8 @@ public class RenameColumnTask extends BaseTableTask { case DERBY_EMBEDDED: sql = "RENAME COLUMN " + getTableName() + "." + myOldName + " TO " + myNewName; break; - case MARIADB_10_1: - sql = "ALTER TABLE " + getTableName() + " CHANGE COLUMN " + myOldName + " TO " + myNewName; - break; case MYSQL_5_7: + case MARIADB_10_1: // Quote the column names as "SYSTEM" is a reserved word in MySQL sql = "ALTER TABLE " + getTableName() + " CHANGE COLUMN `" + myOldName + "` `" + myNewName + "` " + theExistingType + " " + theExistingNotNull; break; diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java index b2edd22caf2..ebaf40f78d2 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameIndexTask.java @@ -110,10 +110,10 @@ public class RenameIndexTask extends BaseTableTask { // Drop constraint switch (theDriverType) { case MYSQL_5_7: + case MARIADB_10_1: // Quote the index names as "PRIMARY" is a reserved word in MySQL sql.add("rename index `" + theOldIndexName + "` to `" + theNewIndexName + "`"); break; - case MARIADB_10_1: case DERBY_EMBEDDED: sql.add("rename index " + theOldIndexName + " to " + theNewIndexName); break; diff --git a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java index 9c4fec4fe74..cd88accb134 100644 --- a/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java +++ b/hapi-fhir-jpaserver-migrate/src/test/java/ca/uhn/fhir/jpa/migrate/taskdef/RenameColumnTaskDbSpecificTest.java @@ -27,7 +27,7 @@ public class RenameColumnTaskDbSpecificTest { @Test public void testBuildSqlStatementForMariaDB() { - assertEquals("ALTER TABLE SOMETABLE CHANGE COLUMN myTextCol TO TEXTCOL", createRenameColumnSql(DriverTypeEnum.MARIADB_10_1)); + assertEquals("ALTER TABLE SOMETABLE CHANGE COLUMN `myTextCol` `TEXTCOL` integer null", createRenameColumnSql(DriverTypeEnum.MARIADB_10_1)); } @Test From dac516828e6fe43bc94999d2108b0b50c7c0108d Mon Sep 17 00:00:00 2001 From: jamesagnew <jamesagnew@gmail.com> Date: Fri, 26 Jun 2020 18:31:13 -0400 Subject: [PATCH 21/22] Move validation settings bean --- .../src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java | 6 ++++++ .../java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 9d1243f4880..e7545b326ab 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -52,6 +52,7 @@ import ca.uhn.fhir.jpa.search.reindex.ResourceReindexingSvcImpl; import ca.uhn.fhir.jpa.searchparam.config.SearchParamConfig; import ca.uhn.fhir.jpa.searchparam.extractor.IResourceLinkResolver; import ca.uhn.fhir.jpa.util.MemoryCacheService; +import ca.uhn.fhir.jpa.validation.ValidationSettings; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.interceptor.consent.IConsentContextServices; import ca.uhn.fhir.rest.server.interceptor.partition.RequestTenantPartitionInterceptor; @@ -213,6 +214,11 @@ public abstract class BaseConfig { return new NpmJpaValidationSupport(); } + @Bean + public ValidationSettings validationSettings() { + return new ValidationSettings(); + } + @Bean public ISearchCacheSvc searchCacheSvc() { return new DatabaseSearchCacheSvcImpl(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java index 46a56e1aedf..04cc6df4ee1 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfigDstu3Plus.java @@ -96,12 +96,6 @@ public abstract class BaseConfigDstu3Plus extends BaseConfig { return val; } - @Bean - public ValidationSettings validationSettings() { - return new ValidationSettings(); - } - - @Bean public abstract ITermReadSvc terminologyService(); From bf8de8480177e36ffe48af27e133a2c3d1c9d093 Mon Sep 17 00:00:00 2001 From: jamesagnew <jamesagnew@gmail.com> Date: Fri, 26 Jun 2020 19:25:36 -0400 Subject: [PATCH 22/22] License header --- ...EmpiCandidateSearchCriteriaBuilderSvc.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java index 50289f0b69e..e2ba0714119 100644 --- a/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java +++ b/hapi-fhir-jpaserver-empi/src/main/java/ca/uhn/fhir/jpa/empi/svc/EmpiCandidateSearchCriteriaBuilderSvc.java @@ -1,5 +1,25 @@ package ca.uhn.fhir.jpa.empi.svc; +/*- + * #%L + * HAPI FHIR JPA Server - Enterprise Master Patient Index + * %% + * Copyright (C) 2014 - 2020 University Health Network + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + import ca.uhn.fhir.empi.rules.json.EmpiResourceSearchParamJson; import org.hl7.fhir.instance.model.api.IAnyResource; import org.springframework.beans.factory.annotation.Autowired;