get sql viewDefinition implementation passing all tests

This commit is contained in:
Grahame Grieve 2023-10-30 23:59:35 +11:00
parent 034b1057e6
commit e4a1a556a2
10 changed files with 248 additions and 120 deletions

View File

@ -730,4 +730,8 @@ public class ExpressionNode {
return names;
}
public boolean isNullSet() {
return kind == Kind.Constant && constant == null;
}
}

View File

@ -13,6 +13,12 @@ public class Cell {
this.column = column;
}
public Cell(Column column, Value value) {
super();
this.column = column;
this.values.add(value);
}
public Column getColumn() {
return column;
}

View File

@ -7,6 +7,7 @@ public class Column {
private String type;
private ColumnKind kind;
private boolean isColl;
private boolean duplicateReported;
protected Column() {
super();
@ -58,5 +59,41 @@ public class Column {
this.isColl = isColl;
}
public String diff(Column other) {
if (!name.equals(other.name)) {
return "Names differ: '"+name+"' vs '"+other.name+"'";
}
if (kind != ColumnKind.Null && other.kind != ColumnKind.Null) {
if (length != other.length) {
return "Lengths differ: '"+length+"' vs '"+other.length+"'";
}
if (kind != other.kind) {
return "Kinds differ: '"+kind+"' vs '"+other.kind+"'";
}
if (isColl != other.isColl) {
return "Collection status differs: '"+isColl+"' vs '"+other.isColl+"'";
}
} else if (kind == ColumnKind.Null) {
kind = other.kind;
length = other.length;
isColl = other.isColl;
}
return null;
}
public boolean isDuplicateReported() {
return duplicateReported;
}
public void setDuplicateReported(boolean duplicateReported) {
this.duplicateReported = duplicateReported;
}
@Override
public String toString() {
return "Column [name=" + name + ", length=" + length + ", type=" + type + ", kind=" + kind + ", isColl=" + isColl
+ "]";
}
}

View File

@ -1,5 +1,5 @@
package org.hl7.fhir.r5.utils.sql;
public enum ColumnKind {
String, DateTime, Integer, Decimal, Binary, Time, Boolean, Complex
String, DateTime, Integer, Decimal, Binary, Time, Boolean, Complex, Null
}

View File

@ -7,5 +7,5 @@ import org.hl7.fhir.r5.model.Base;
public interface Provider {
List<Base> fetch(String resourceType);
Base resolveReference(String ref, String resourceType);
Base resolveReference(Base rootResource, String ref, String specifiedResourceType);
}

View File

@ -24,6 +24,7 @@ import org.hl7.fhir.r5.utils.FHIRPathEngine;
import org.hl7.fhir.r5.utils.FHIRPathEngine.IEvaluationContext;
import org.hl7.fhir.r5.utils.FHIRPathUtilityClasses.FunctionDetails;
import org.hl7.fhir.utilities.json.model.JsonObject;
import org.hl7.fhir.utilities.validation.ValidationMessage;
public class Runner implements IEvaluationContext {
@ -33,9 +34,8 @@ public class Runner implements IEvaluationContext {
private List<String> prohibitedNames = new ArrayList<String>();
private FHIRPathEngine fpe;
private List<Column> columns = new ArrayList<>();
private String resourceName;
private List<ValidationMessage> issues;
public IWorkerContext getContext() {
@ -86,15 +86,15 @@ public class Runner implements IEvaluationContext {
}
Validator validator = new Validator(context, fpe, prohibitedNames, storage.supportsArrays(), storage.supportsComplexTypes(), storage.needsName());
validator.checkViewDefinition(path, viewDefinition);
issues = validator.getIssues();
validator.dump();
validator.check();
resourceName = validator.getResourceName();
columns = validator.getColumns();
evaluate(viewDefinition);
}
private void evaluate(JsonObject vd) {
Store store = storage.createStore(vd.asString("name"), columns);
Store store = storage.createStore(vd.asString("name"), (List<Column>) vd.getUserData("columns"));
List<Base> data = provider.fetch(resourceName);
@ -130,7 +130,20 @@ public class Runner implements IEvaluationContext {
if (select.has("forEach")) {
focus.addAll(executeForEach(select, b));
} else if (select.has("forEachOrNull")) {
focus.addAll(executeForEachOrNull(select, b));
if (focus.isEmpty()) {
List<Column> columns = (List<Column>) select.getUserData("columns");
for (List<Cell> row : rows) {
for (Column c : columns) {
Cell cell = cell(row, c.getName());
if (cell == null) {
row.add(new Cell(c, null));
}
}
}
return;
}
} else {
focus.add(b);
}
@ -149,19 +162,30 @@ public class Runner implements IEvaluationContext {
executeColumn(column, f, rowsToAdd);
}
for (JsonObject sub : select.getJsonObjects("unionAll")) {
executeSelect(sub, f, rowsToAdd);
}
for (JsonObject sub : select.getJsonObjects("select")) {
executeSelect(sub, f, rowsToAdd);
}
executeUnionAll(select.getJsonObjects("unionAll"), f, rowsToAdd);
rows.addAll(rowsToAdd);
}
}
private List<Base> executeUnion(JsonObject focus, Base b, List<List<Cell>> rows) {
throw new FHIRException("union is not supported");
private void executeUnionAll(List<JsonObject> unionList, Base b, List<List<Cell>> rows) {
if (unionList.isEmpty()) {
return;
}
List<List<Cell>> sourceRows = new ArrayList<>();
sourceRows.addAll(rows);
rows.clear();
for (JsonObject union : unionList) {
List<List<Cell>> tempRows = new ArrayList<>();
tempRows.addAll(sourceRows);
executeSelect(union, b, tempRows);
rows.addAll(tempRows);
}
}
private List<List<Cell>> cloneRows(List<List<Cell>> rows) {
@ -191,9 +215,6 @@ public class Runner implements IEvaluationContext {
ExpressionNode n = (ExpressionNode) focus.getUserData("forEachOrNull");
List<Base> result = new ArrayList<>();
result.addAll(fpe.evaluate(b, n));
if (result.size() == 0) {
result.add(null);
}
return result;
}
@ -203,11 +224,14 @@ public class Runner implements IEvaluationContext {
if (b != null) {
bl2.addAll(fpe.evaluate(b, n));
}
String name = column.getUserString("name");
Column col = (Column) column.getUserData("column");
if (col == null) {
System.out.println("Error");
} else {
for (List<Cell> row : rows) {
Cell c = cell(row, name);
Cell c = cell(row, col.getName());
if (c == null) {
c = new Cell(column(name));
c = new Cell(col);
row.add(c);
}
if (!bl2.isEmpty()) {
@ -224,6 +248,7 @@ public class Runner implements IEvaluationContext {
}
}
}
}
private Value genValue(Column column, Base b) {
@ -299,7 +324,7 @@ public class Runner implements IEvaluationContext {
}
}
private Column column(String columnName) {
private Column column(String columnName, List<Column> columns) {
for (Column t : columns) {
if (t.getName().equalsIgnoreCase(columnName)) {
return t;
@ -353,7 +378,7 @@ public class Runner implements IEvaluationContext {
public List<Base> executeFunction(FHIRPathEngine engine, Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
switch (functionName) {
case "getResourceKey" : return executeResourceKey(focus);
case "getReferenceKey" : return executeReferenceKey(focus, parameters);
case "getReferenceKey" : return executeReferenceKey(null, focus, parameters);
default: throw new Error("Not known: "+functionName);
}
}
@ -375,7 +400,7 @@ public class Runner implements IEvaluationContext {
return base;
}
private List<Base> executeReferenceKey(List<Base> focus, List<List<Base>> parameters) {
private List<Base> executeReferenceKey(Base rootResource, List<Base> focus, List<List<Base>> parameters) {
String rt = null;
if (parameters.size() > 0) {
rt = parameters.get(0).get(0).primitiveValue();
@ -395,7 +420,7 @@ public class Runner implements IEvaluationContext {
throw new FHIRException("Unable to generate a reference key based on a "+res.fhirType());
}
if (ref != null) {
Base target = provider.resolveReference(ref, rt);
Base target = provider.resolveReference(rootResource, ref, rt);
if (target != null) {
if (!res.hasUserData("Storage.key")) {
String key = storage.getKeyForTargetResource(target);
@ -437,6 +462,9 @@ public class Runner implements IEvaluationContext {
public boolean paramIsType(String name, int index) {
return "getReferenceKey".equals(name);
}
public List<ValidationMessage> getIssues() {
return issues;
}
}

View File

@ -48,7 +48,9 @@ public class StorageJson implements Storage {
}
private JsonElement makeJsonNode(Value value) {
if (value.getValueInt() != null) {
if (value == null) {
return new JsonNull();
} else if (value.getValueInt() != null) {
return new JsonNumber(value.getValueInt().intValue());
}
if (value.getValueBoolean() != null) {

View File

@ -2,6 +2,7 @@ package org.hl7.fhir.r5.utils.sql;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLType;
import java.util.List;
import org.hl7.fhir.exceptions.FHIRException;
@ -27,7 +28,7 @@ public class StorageSqlite3 implements Storage {
private Connection conn;
private int nextKey = 0;
protected StorageSqlite3(Connection conn) {
public StorageSqlite3(Connection conn) {
super();
this.conn = conn;
}
@ -38,7 +39,7 @@ public class StorageSqlite3 implements Storage {
CommaSeparatedStringBuilder fields = new CommaSeparatedStringBuilder(", ");
CommaSeparatedStringBuilder values = new CommaSeparatedStringBuilder(", ");
StringBuilder b = new StringBuilder();
b.append("Create Table "+name+" { ");
b.append("Create Table "+name+" ( ");
b.append("ViewRowKey integer NOT NULL");
for (Column column : columns) {
b.append(", "+column.getName()+" "+sqliteType(column.getKind())+" NULL"); // index columns are always nullable
@ -48,7 +49,7 @@ public class StorageSqlite3 implements Storage {
b.append(", PRIMARY KEY (ViewRowKey))\r\n");
conn.createStatement().execute(b.toString());
String isql = "Insert into "+name+" ("+fields.toString()+") values ("+values.toString()+")";
String isql = "Insert into "+name+" (ViewRowKey, "+fields.toString()+") values (?, "+values.toString()+")";
PreparedStatement psql = conn.prepareStatement(isql);
return new SQLiteStore(name, psql);
} catch (Exception e) {
@ -79,6 +80,8 @@ public class StorageSqlite3 implements Storage {
for (int i = 0; i < cells.size(); i++) {
Cell c = cells.get(i);
switch (c.getColumn().getKind()) {
case Null:
p.setNull(i+2, java.sql.Types.NVARCHAR);
case Binary:
p.setBytes(i+2, c.getValues().size() == 0 ? null : c.getValues().get(0).getValueBinary());
break;
@ -103,6 +106,7 @@ public class StorageSqlite3 implements Storage {
case Complex: throw new FHIRException("SQLite runner does not handle complexes");
}
}
p.execute();
} catch (Exception e) {
throw new FHIRException(e);
}

View File

@ -18,6 +18,7 @@ import org.hl7.fhir.utilities.json.model.JsonBoolean;
import org.hl7.fhir.utilities.json.model.JsonElement;
import org.hl7.fhir.utilities.json.model.JsonNumber;
import org.hl7.fhir.utilities.json.model.JsonObject;
import org.hl7.fhir.utilities.json.model.JsonProperty;
import org.hl7.fhir.utilities.json.model.JsonString;
import org.hl7.fhir.utilities.validation.ValidationMessage;
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
@ -35,7 +36,6 @@ public class Validator {
private Boolean needsName;
private String resourceName;
private List<Column> columns = new ArrayList<Column>();
private String name;
public Validator(IWorkerContext context, FHIRPathEngine fpe, List<String> prohibitedNames, Boolean arrays, Boolean complexTypes, Boolean needsName) {
@ -52,11 +52,10 @@ public class Validator {
return resourceName;
}
public List<Column> getColumns() {
return columns;
}
public void checkViewDefinition(String path, JsonObject viewDefinition) {
checkProperties(viewDefinition, path, "url", "identifier", "name", "version", "title", "status", "experimental", "date", "publisher", "contact", "description", "useContext", "copyright", "resource", "constant", "select", "where");
JsonElement nameJ = viewDefinition.get("name");
if (nameJ == null) {
if (needsName == null) {
@ -76,9 +75,12 @@ public class Validator {
}
}
List<Column> columns = new ArrayList<>();
viewDefinition.setUserData("columns", columns);
JsonElement resourceNameJ = viewDefinition.get("resource");
if (resourceNameJ == null) {
error(path, viewDefinition, "No resource provided", IssueType.REQUIRED);
error(path, viewDefinition, "No resource specified", IssueType.REQUIRED);
} else if (!(resourceNameJ instanceof JsonString)) {
error(path, viewDefinition, "resource must be a string", IssueType.INVALID);
} else {
@ -102,17 +104,10 @@ public class Validator {
}
TypeDetails t = new TypeDetails(CollectionStatus.SINGLETON, resourceName);
if (viewDefinition.has("forEach")) {
checkForEach(path, viewDefinition, viewDefinition.get("forEach"), t);
} else if (viewDefinition.has("forEachOrNull")) {
checkForEachOrNull(path, viewDefinition, viewDefinition.get("forEachOrNull"), t);
} else if (viewDefinition.has("unionAll")) {
checkUnion(path, viewDefinition, viewDefinition.get("unionAll"), t);
} else {
i = 0;
if (checkAllObjects(path, viewDefinition, "select")) {
for (JsonObject select : viewDefinition.getJsonObjects("select")) {
checkSelect(path+".select["+i+"]", select, t);
columns.addAll(checkSelect(path+".select["+i+"]", select, t));
i++;
}
if (i == 0) {
@ -122,9 +117,11 @@ public class Validator {
}
}
}
}
private void checkSelect(String path, JsonObject select, TypeDetails t) {
private List<Column> checkSelect(String path, JsonObject select, TypeDetails t) {
List<Column> columns = new ArrayList<>();
select.setUserData("columns", columns);
checkProperties(select, path, "column", "select", "forEach", "forEachOrNull", "unionAll");
if (select.has("forEach")) {
t = checkForEach(path, select, select.get("forEach"), t);
@ -133,24 +130,18 @@ public class Validator {
}
if (t != null) {
boolean content = false;
if (select.has("unionAll")) {
content = checkUnion(path, select, select.get("unionAll"), t);
}
if (select.has("column")) {
JsonElement a = select.get("column");
if (!(a instanceof JsonArray)) {
error(path+".column", a, "column is not an array", IssueType.INVALID);
} else {
content = true;
int i = 0;
for (JsonElement e : ((JsonArray) a)) {
if (!(e instanceof JsonObject)) {
error(path+".column["+i+"]", a, "column["+i+"] is a "+e.type().toName()+" not an object", IssueType.INVALID);
} else {
checkColumn(path+".column["+i+"]", (JsonObject) e, t);
columns.add(checkColumn(path+".column["+i+"]", (JsonObject) e, t));
}
}
}
@ -161,47 +152,98 @@ public class Validator {
if (!(a instanceof JsonArray)) {
error(path+".select", a, "select is not an array", IssueType.INVALID);
} else {
content = true;
int i = 0;
for (JsonElement e : ((JsonArray) a)) {
if (!(e instanceof JsonObject)) {
error(path+".select["+i+"]", e, "select["+i+"] is not an object", IssueType.INVALID);
} else {
checkSelect(path+".select["+i+"]", (JsonObject) e, t);
columns.addAll(checkSelect(path+".select["+i+"]", (JsonObject) e, t));
}
}
}
}
if (!content) {
if (select.has("unionAll")) {
columns.addAll(checkUnion(path, select, select.get("unionAll"), t));
}
if (columns.isEmpty()) {
error(path, select, "The select has no columns or selects", IssueType.REQUIRED);
} else {
checkColumnNamesUnique(select, path, columns);
}
}
return columns;
}
private void checkColumnNamesUnique(JsonObject select, String path, List<Column> columns) {
Set<String> names = new HashSet<>();
for (Column col : columns) {
if (col != null) {
if (!names.contains(col.getName())) {
names.add(col.getName());
} else if (!col.isDuplicateReported()) {
col.setDuplicateReported(true);
error(path, select, "Duplicate Column Name '"+col.getName()+"'", IssueType.BUSINESSRULE);
}
}
}
}
private boolean checkUnion(String path, JsonObject focus, JsonElement expression, TypeDetails t) {
private List<Column> checkUnion(String path, JsonObject focus, JsonElement expression, TypeDetails t) {
JsonElement a = focus.get("unionAll");
if (!(a instanceof JsonArray)) {
error(path+".union", a, "union is not an array", IssueType.INVALID);
return false;
error(path+".unionAll", a, "union is not an array", IssueType.INVALID);
return null;
} else {
List<List<Column>> unionColumns = new ArrayList<>();
int i = 0;
for (JsonElement e : ((JsonArray) a)) {
if (!(e instanceof JsonObject)) {
error(path+".union["+i+"]", e, "union["+i+"] is not an object", IssueType.INVALID);
error(path+".unionAll["+i+"]", e, "unionAll["+i+"] is not an object", IssueType.INVALID);
} else {
checkSelect(path+".union["+i+"]", (JsonObject) e, t);
unionColumns.add(checkSelect(path+".unionAll["+i+"]", (JsonObject) e, t));
}
i++;
}
if (i < 2) {
warning(path+".union", a, "union should have more than one item");
warning(path+".unionAll", a, "unionAll should have more than one item");
}
return true;
if (unionColumns.size() > 1) {
List<Column> columns = unionColumns.get(0);
for (int ic = 1; ic < unionColumns.size(); ic++) {
String diff = columnDiffs(columns, unionColumns.get(ic));
if (diff != null) {
error(path+".unionAll["+i+"]", ((JsonArray) a).get(ic), "unionAll["+i+"] column definitions do not match: "+diff, IssueType.INVALID);
}
}
a.setUserData("colunms", columns);
return columns;
}
}
return null;
}
private String columnDiffs(List<Column> list1, List<Column> list2) {
if (list1.size() == list2.size()) {
for (int i = 0; i < list1.size(); i++) {
if (list1.get(i) == null || list2.get(i) == null) {
return null; // just suppress any addition errors
}
String diff = list1.get(i).diff(list2.get(i));
if (diff != null) {
return diff+" at #"+i;
}
}
return null;
} else {
return "Column counts differ: "+list1.size()+" vs "+list2.size();
}
}
private void checkColumn(String path, JsonObject column, TypeDetails t) {
private Column checkColumn(String path, JsonObject column, TypeDetails t) {
checkProperties(column, path, "path", "name", "description", "collection", "type", "tag");
if (!column.has("path")) {
error(path, column, "no path found", IssueType.INVALID);
} else {
@ -253,7 +295,7 @@ public class Validator {
// ok, name is sorted!
if (columnName != null) {
column.setUserData("name", columnName);
boolean isColl = (td.getCollectionStatus() != CollectionStatus.SINGLETON) || column(columnName) != null;
boolean isColl = (td.getCollectionStatus() != CollectionStatus.SINGLETON);
if (column.has("collection")) {
JsonElement collectionJ = column.get("collection");
if (!(collectionJ instanceof JsonBoolean)) {
@ -273,8 +315,11 @@ public class Validator {
warning(path, expression, "column appears to be a collection, but this is not allowed in this context");
}
}
// ok collection is sorted
Set<String> types = new HashSet<>();
if (node.isNullSet()) {
types.add("null");
} else {
// ok collection is sorted
for (String type : td.getTypes()) {
types.add(simpleType(type));
}
@ -293,12 +338,13 @@ public class Validator {
error(path+".type", typeJ, "type must be a string", IssueType.INVALID);
}
}
}
if (types.size() != 1) {
error(path, column, "Unable to determine a type (found "+td.describe()+")", IssueType.BUSINESSRULE);
} else {
String type = types.iterator().next();
boolean ok = false;
if (!isSimpleType(type)) {
if (!isSimpleType(type) && !"null".equals(type)) {
if (complexTypes) {
warning(path, expression, "Column is a complex type. This is not supported in some Runners");
} else if (!complexTypes) {
@ -310,28 +356,21 @@ public class Validator {
ok = true;
}
if (ok) {
Column col = column(columnName);
if (col != null) {
if (!col.getType().equals(type)) {
error(path, expression, "Duplicate definition for "+columnName+" has different types ("+col.getType()+" vs "+type+")", IssueType.BUSINESSRULE);
}
if (col.isColl() != isColl) {
error(path, expression, "Duplicate definition for "+columnName+" has different status for collection ("+col.isColl()+" vs "+isColl+")", IssueType.BUSINESSRULE);
}
} else {
columns.add(new Column(columnName, isColl, type, kindForType(type)));
}
Column col = new Column(columnName, isColl, type, kindForType(type));
column.setUserData("column", col);
return col;
}
}
}
}
}
}
return null;
}
private ColumnKind kindForType(String type) {
switch (type) {
case "null": return ColumnKind.Null;
case "dateTime": return ColumnKind.DateTime;
case "boolean": return ColumnKind.Boolean;
case "integer": return ColumnKind.Integer;
@ -343,15 +382,6 @@ public class Validator {
}
}
private Column column(String columnName) {
for (Column t : columns) {
if (t.getName().equalsIgnoreCase(columnName)) {
return t;
}
}
return null;
}
private boolean isSimpleType(String type) {
return Utilities.existsInList(type, "dateTime", "boolean", "integer", "decimal", "string", "base64Binary");
}
@ -433,6 +463,7 @@ public class Validator {
}
private void checkConstant(String path, JsonObject constant) {
checkProperties(constant, path, "name", "valueBase64Binary", "valueBoolean", "valueCanonical", "valueCode", "valueDate", "valueDateTime", "valueDecimal", "valueId", "valueInstant", "valueInteger", "valueInteger64", "valueOid", "valueString", "valuePositiveInt", "valueTime", "valueUnsignedInt", "valueUri", "valueUrl", "valueUuid");
JsonElement nameJ = constant.get("name");
if (nameJ == null) {
error(path, constant, "No name provided", IssueType.REQUIRED);
@ -508,6 +539,8 @@ public class Validator {
}
}
private void checkWhere(String path, JsonObject where) {
checkProperties(where, path, "path", "description");
String expr = where.asString("path");
if (expr == null) {
error(path, where, "No path provided", IssueType.REQUIRED);
@ -534,6 +567,19 @@ public class Validator {
}
}
private void checkProperties(JsonObject obj, String path, String... names) {
for (JsonProperty p : obj.getProperties()) {
boolean nameOk = "extension".equals(p.getName());
for (String name : names) {
nameOk = nameOk || name.equals(p.getName());
}
if (!nameOk) {
error(path+"."+p.getName(), p.getValue(), "Unknown JSON "+p.getValue().type().name(), IssueType.UNKNOWN);
}
}
}
private boolean isValidName(String name) {
boolean first = true;
for (char c : name.toCharArray()) {

View File

@ -54,7 +54,7 @@ public class SQLOnFhirTests {
}
@Override
public Base resolveReference(String ref, String rt) {
public Base resolveReference(Base rootResource, String ref, String rt) {
if (ref == null) {
return null;
}
@ -109,7 +109,7 @@ public class SQLOnFhirTests {
@SuppressWarnings("deprecation")
@ParameterizedTest(name = "{index}: file {0}")
@MethodSource("data")
@Disabled
//@Disabled
public void test(String name, TestDetails test) throws FileNotFoundException, IOException, FHIRException, org.hl7.fhir.exceptions.FHIRException, UcumException {
this.details = test;
Runner runner = new Runner();
@ -126,6 +126,7 @@ public class SQLOnFhirTests {
runner.execute(test.path+".view", test.testCase.getJsonObject("view"));
results = store.getRows();
} catch (Exception e) {
e.printStackTrace();
Assertions.assertTrue(test.testCase.has("expectError"), e.getMessage());
}
if (results != null) {