From 0833c904241cedbc3a20573a7ff9dd751f440144 Mon Sep 17 00:00:00 2001 From: michaelpede Date: Tue, 17 Aug 2021 12:26:35 -0700 Subject: [PATCH 1/2] Add DDL for Field table. Also, a new build process that uses docker to build parts that fail on Macs --- .env | 8 +- build.sh | 38 +-- docker/docker-builder | 7 + docker/scripts/build.sh | 42 ++++ .../GenericEntityCollectionProcessor.java | 11 +- .../service/data/GenericEntityProcessor.java | 10 +- .../definition/custom/FieldDefinition.java | 221 ++++++++++++++++++ .../BreakdownOfFilterExpressionVisitor.java | 179 ++++++++++++++ .../org/reso/service/data/meta/FieldInfo.java | 3 +- .../reso/service/data/meta/ResourceInfo.java | 18 ++ .../org/reso/service/servlet/RESOservlet.java | 33 +-- 11 files changed, 513 insertions(+), 57 deletions(-) create mode 100644 docker/docker-builder create mode 100644 docker/scripts/build.sh create mode 100644 src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java create mode 100644 src/main/java/org/reso/service/data/meta/BreakdownOfFilterExpressionVisitor.java diff --git a/.env b/.env index d31430a..d9c3320 100644 --- a/.env +++ b/.env @@ -2,7 +2,7 @@ COMPOSE__FILE=docker-compose.yml:./optional/docker-db-compose.yml SQL_HOST=192.168.1.71 SQL_USER=mpede SQL_PASSWORD=password -#SQL_DB_DRIVER=com.mysql.cj.jdbc.Driver -SQL_DB_DRIVER=org.postgresql.Driver -#SQL_CONNECTION_STR=jdbc:mysql://192.168.1.71/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4&user=mpede&password=password -SQL_CONNECTION_STR=jdbc:postgresql://192.168.1.71:5432/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4 +SQL_DB_DRIVER=com.mysql.cj.jdbc.Driver +#SQL_DB_DRIVER=org.postgresql.Driver +SQL_CONNECTION_STR=jdbc:mysql://192.168.1.71/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4&user=mpede&password=password +#SQL_CONNECTION_STR=jdbc:postgresql://192.168.1.71:5432/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4 diff --git a/build.sh b/build.sh index eb26827..c382e22 100644 --- a/build.sh +++ b/build.sh @@ -5,43 +5,11 @@ HOME_DIR=`dirname ${REAL_VAR0}` TEMP_DIR="${HOME_DIR}/temp" SQL_DIR="${HOME_DIR}/sql" -# Ensure we have directories set up -[ -d "${TEMP_DIR}" ] && echo "temp directory found." || mkdir ${TEMP_DIR} -[ -d "${SQL_DIR}" ] && echo "sql directory found." || mkdir ${SQL_DIR} - -if [ -z "${SQL_HOST}" ] -then - # Get the Web API Commander, needed to generate the test database - if ! wget https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar -O temp/web-api-commander.jar - then - echo "WGET not installed. trying CURL." - if ! curl -L https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar --output temp/web-api-commander.jar - then - echo "CURL not installed. Exiting build." - exit - fi - fi - - java -jar temp/web-api-commander.jar --generateReferenceDDL --useKeyNumeric > sql/reso-reference-ddl-dd-1.7.numeric-keys.sql - - # The following lines should be independent of the SQL logic. - java -jar temp/web-api-commander.jar --generateResourceInfoModels - mv ResourceInfoModels/* src/main/java/org/reso/service/data/definition/ - -else - COMPOSE_FILE="docker-compose.yml" -fi - -if ! mvn compile -then - echo "Maven could not be found." - exit -else - mvn package -fi +docker build -t reso-builder -f docker/docker-builder . +docker run --name builder --mount type=bind,source="${HOME_DIR}",target=/usr/src/app -t reso-builder if ! docker-compose build then echo "docker-compose could not be found. You may need to install with pip." exit -fi +fi \ No newline at end of file diff --git a/docker/docker-builder b/docker/docker-builder new file mode 100644 index 0000000..e64a568 --- /dev/null +++ b/docker/docker-builder @@ -0,0 +1,7 @@ +FROM ubuntu:20.10 + +WORKDIR /usr/src/app + +RUN apt-get update && apt-get install -y wget curl pip maven docker-compose + +CMD ./docker/scripts/build.sh diff --git a/docker/scripts/build.sh b/docker/scripts/build.sh new file mode 100644 index 0000000..8860342 --- /dev/null +++ b/docker/scripts/build.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +REAL_VAR0=`readlink -f $0` +HOME_DIR=`dirname ${REAL_VAR0}` +TEMP_DIR="${HOME_DIR}/temp" +SQL_DIR="${HOME_DIR}/sql" + +# Ensure we have directories set up +[ -d "${TEMP_DIR}" ] && echo "temp directory found." || mkdir ${TEMP_DIR} +[ -d "${SQL_DIR}" ] && echo "sql directory found." || mkdir ${SQL_DIR} + +if [ -z "${SQL_HOST}" ] +then + # Get the Web API Commander, needed to generate the test database + if ! wget https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar -O temp/web-api-commander.jar + then + echo "WGET not installed. trying CURL." + if ! curl -L https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar --output temp/web-api-commander.jar + then + echo "CURL not installed. Exiting build." + exit + fi + fi + + java -jar temp/web-api-commander.jar --generateReferenceDDL --useKeyNumeric > sql/reso-reference-ddl-dd-1.7.numeric-keys.sql + + # The following lines should be independent of the SQL logic. + java -jar temp/web-api-commander.jar --generateResourceInfoModels + mv ResourceInfoModels/* src/main/java/org/reso/service/data/definition/ + +else + COMPOSE_FILE="docker-compose.yml" +fi + +if ! mvn compile +then + echo "Maven could not be found." + exit +else + mvn package +fi + diff --git a/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java b/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java index db90eba..aa6faf1 100644 --- a/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java +++ b/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java @@ -85,7 +85,16 @@ public class GenericEntityCollectionProcessor implements EntityCollectionProcess // 2nd: fetch the data from backend for this requested EntitySetName // it has to be delivered as EntitySet object - EntityCollection entitySet = getData(edmEntitySet, uriInfo, isCount, resource); + EntityCollection entitySet; + + if (resource.useCustomDatasource() ) + { + entitySet = resource.getData(edmEntitySet, uriInfo, isCount); + } + else + { + entitySet = getData(edmEntitySet, uriInfo, isCount, resource); + } // 3rd: create a serializer based on the requested format (json) try diff --git a/src/main/java/org/reso/service/data/GenericEntityProcessor.java b/src/main/java/org/reso/service/data/GenericEntityProcessor.java index f46a90b..abc6f6e 100644 --- a/src/main/java/org/reso/service/data/GenericEntityProcessor.java +++ b/src/main/java/org/reso/service/data/GenericEntityProcessor.java @@ -65,7 +65,15 @@ public class GenericEntityProcessor implements EntityProcessor // 2. retrieve the data from backend List keyPredicates = uriResourceEntitySet.getKeyPredicates(); - Entity entity = getData(edmEntitySet, keyPredicates, resource); + Entity entity; + if (resource.useCustomDatasource() ) + { + entity = resource.getData(edmEntitySet, keyPredicates); + } + else + { + entity = getData(edmEntitySet, keyPredicates, resource); + } // 3. serialize EdmEntityType entityType = edmEntitySet.getEntityType(); diff --git a/src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java b/src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java new file mode 100644 index 0000000..2ba0405 --- /dev/null +++ b/src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java @@ -0,0 +1,221 @@ +package org.reso.service.data.definition.custom; + +import org.apache.olingo.commons.api.data.Entity; +import org.apache.olingo.commons.api.data.EntityCollection; +import org.apache.olingo.commons.api.data.Property; +import org.apache.olingo.commons.api.data.ValueType; +import org.apache.olingo.commons.api.edm.EdmEntitySet; +import org.apache.olingo.commons.api.edm.EdmPrimitiveTypeKind; +import org.apache.olingo.commons.api.edm.provider.CsdlAnnotation; +import org.apache.olingo.commons.api.edm.provider.CsdlEntitySet; +import org.apache.olingo.server.api.ODataApplicationException; +import org.apache.olingo.server.api.uri.UriInfo; +import org.apache.olingo.server.api.uri.UriParameter; +import org.apache.olingo.server.api.uri.queryoption.FilterOption; +import org.apache.olingo.server.api.uri.queryoption.expression.ExpressionVisitException; +import org.reso.service.data.common.CommonDataProcessing; +import org.reso.service.data.meta.BreakdownOfFilterExpressionVisitor; +import org.reso.service.data.meta.FieldInfo; +import org.reso.service.data.meta.MySQLFilterExpressionVisitor; +import org.reso.service.data.meta.ResourceInfo; + +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.*; + +public class FieldDefinition extends ResourceInfo +{ + private static String STANDARD_NAME = "RESO.OData.Metadata.StandardName"; + + private static ArrayList fieldList = null; + private ArrayList resources; + + public FieldDefinition() + { + this.tableName = "field"; // Never used + this.resourcesName = "Field"; + this.resourceName = "Field"; + } + + public ArrayList getFieldList() + { + return FieldDefinition.getStaticFieldList(); + } + + public static ArrayList getStaticFieldList() + { + if (null!= FieldDefinition.fieldList) + { + return FieldDefinition.fieldList; + } + + ArrayList list = new ArrayList(); + FieldDefinition.fieldList = list; + FieldInfo fieldInfo = null; + + fieldInfo = new FieldInfo("FieldKey", EdmPrimitiveTypeKind.String.getFullQualifiedName()); + fieldInfo.addAnnotation("Field Key Field", "RESO.OData.Metadata.DisplayName"); + list.add(fieldInfo); + + fieldInfo = new FieldInfo("ResourceName", EdmPrimitiveTypeKind.String.getFullQualifiedName()); + list.add(fieldInfo); + + fieldInfo = new FieldInfo("FieldName", EdmPrimitiveTypeKind.String.getFullQualifiedName()); + list.add(fieldInfo); + + fieldInfo = new FieldInfo("DisplayName", EdmPrimitiveTypeKind.String.getFullQualifiedName()); + list.add(fieldInfo); + + fieldInfo = new FieldInfo("ModificationTimestamp", EdmPrimitiveTypeKind.DateTimeOffset.getFullQualifiedName()); + list.add(fieldInfo); + + return FieldDefinition.fieldList; + } + + public Boolean useCustomDatasource() { return true; } + + public Entity getData(EdmEntitySet edmEntitySet, List keyPredicates) + { + ArrayList fields = this.getFieldList(); + + Entity product = null; + + Map properties = System.getenv(); + + try { + + String sqlCriteria = null; + + // Statements allow to issue SQL queries to the database + Statement statement = null; + // Result set get the result of the SQL query + String queryString = null; + + for (final UriParameter key : keyPredicates) + { + // key + String keyName = key.getName().toLowerCase(); + String keyValue = key.getText(); + if (sqlCriteria==null) + { + sqlCriteria = keyName + " = " + keyValue; + } + else + { + sqlCriteria = sqlCriteria + " and " + keyName + " = " + keyValue; + } + } + + queryString = "select * from " + this.getTableName(); + + if (null!=sqlCriteria && sqlCriteria.length()>0) + { + queryString = queryString + " WHERE " + sqlCriteria; + } + + LOG.info("SQL Query: "+queryString); + ResultSet resultSet = statement.executeQuery(queryString); + + String primaryFieldName = this.getPrimaryKeyName(); + + // add the lookups from the database. + while (resultSet.next()) + { + Entity ent = CommonDataProcessing.getEntityFromRow(resultSet, this, null); + + product = ent; + } + + statement.close(); + + } catch (Exception e) { + LOG.error("Server Error occurred in reading "+this.getResourceName(), e); + return product; + } + + return product; + } + + public EntityCollection getData(EdmEntitySet edmEntitySet, UriInfo uriInfo, boolean isCount) throws ODataApplicationException + { + EntityCollection entCollection = new EntityCollection(); + List productList = entCollection.getEntities(); + + FilterOption filter = uriInfo.getFilterOption(); + + BreakdownOfFilterExpressionVisitor customExpression = new BreakdownOfFilterExpressionVisitor(this); + try + { + String criteria = filter.getExpression().accept(customExpression); + } + catch (ExpressionVisitException e) + { + LOG.error("Server Error occurred in reading "+this.getResourceName(), e); + return entCollection; + } + + HashMap reps = customExpression.getRepresentations(); + + String resourceName = reps.remove("ResourceName"); + resourceName = resourceName.substring(1,resourceName.length()-1); + + ResourceInfo res = null; + + for (ResourceInfo defn :resources) + { + if (resourceName.equals(defn.getResourcesName())) + { + res = defn; + break; + } + } + + ArrayList resourceFieldList = res.getFieldList(); + + for (FieldInfo field: resourceFieldList) + { + HashMap entityValues = new HashMap<>(); + entityValues.put("FieldKey", field.getFieldName()); + entityValues.put("FieldName", field.getFieldName()); + entityValues.put("ResourceName", resourceName); + entityValues.put("DisplayName", field.getFieldName()); + Date date = new Date(); + entityValues.put("ModificationTimestamp", date); + + boolean match = true; + + for (String key: reps.keySet() ) + { + String toMatch = reps.get(key); + toMatch = toMatch.substring(1,toMatch.length()-1); + String value = entityValues.get(key).toString(); + if (!toMatch.equals(value)) + { + match = false; + break; + } + } + + if (match) + { + Entity ent = new Entity(); + ent.addProperty(new Property(null, "FieldKey", ValueType.PRIMITIVE, field.getFieldName())); + ent.addProperty(new Property(null, "FieldName", ValueType.PRIMITIVE, field.getFieldName())); + ent.addProperty(new Property(null, "ResourceName", ValueType.PRIMITIVE, resourceName)); + + String displayName = field.getFieldName(); + ent.addProperty(new Property(null, "DisplayName", ValueType.PRIMITIVE, displayName)); + ent.addProperty(new Property(null, "ModificationTimestamp", ValueType.PRIMITIVE, date )); + productList.add(ent); + } + } + + return entCollection; + } + + public void addResources(ArrayList resources) + { + this.resources = resources; + } +} diff --git a/src/main/java/org/reso/service/data/meta/BreakdownOfFilterExpressionVisitor.java b/src/main/java/org/reso/service/data/meta/BreakdownOfFilterExpressionVisitor.java new file mode 100644 index 0000000..774f0ce --- /dev/null +++ b/src/main/java/org/reso/service/data/meta/BreakdownOfFilterExpressionVisitor.java @@ -0,0 +1,179 @@ +package org.reso.service.data.meta; + + +import org.apache.olingo.commons.api.edm.EdmEnumType; +import org.apache.olingo.commons.api.edm.EdmPrimitiveTypeKind; +import org.apache.olingo.commons.api.edm.EdmType; +import org.apache.olingo.commons.api.http.HttpStatusCode; +import org.apache.olingo.server.api.ODataApplicationException; +import org.apache.olingo.server.api.uri.UriResource; +import org.apache.olingo.server.api.uri.UriResourcePrimitiveProperty; +import org.apache.olingo.server.api.uri.queryoption.expression.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class BreakdownOfFilterExpressionVisitor implements ExpressionVisitor +{ + private static final Logger LOG = LoggerFactory.getLogger(MySQLFilterExpressionVisitor.class); + private static final Map BINARY_OPERATORS = new HashMap() {{ + put(BinaryOperatorKind.ADD, " + "); + put(BinaryOperatorKind.AND, " AND "); + put(BinaryOperatorKind.DIV, " / "); + put(BinaryOperatorKind.EQ, " = "); + put(BinaryOperatorKind.GE, " >= "); + put(BinaryOperatorKind.GT, " > "); + put(BinaryOperatorKind.LE, " <= "); + put(BinaryOperatorKind.LT, " < "); + put(BinaryOperatorKind.MOD, " % "); + put(BinaryOperatorKind.MUL, " * "); + put(BinaryOperatorKind.NE, " <> "); + put(BinaryOperatorKind.OR, " OR "); + put(BinaryOperatorKind.SUB, " - "); + }}; + + private HashMap representations = new HashMap<>(); + + private ResourceInfo resourceInfo; + + public BreakdownOfFilterExpressionVisitor(ResourceInfo resourceInfo) { + this.resourceInfo = resourceInfo; + } + + public HashMap getRepresentations() + { + return representations; + } + + @Override + public String visitBinaryOperator(BinaryOperatorKind operator, String left, String right) + throws ExpressionVisitException, ODataApplicationException + { + String strOperator = BINARY_OPERATORS.get(operator); + + if (strOperator == null) { + throw new ODataApplicationException("Unsupported binary operation: " + operator.name(), + operator == BinaryOperatorKind.HAS ? + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode() : + HttpStatusCode.BAD_REQUEST.getStatusCode(), Locale.ENGLISH); + } + // ResourceName eq 'Property' becomes (=, ResourceName, Property) + if ( "=".equals(strOperator.trim() ) ) // We only want the equals operator from the filter. Assume all are AND ops for now if there are multiples. + { // @TODO: Make this filter more robust for queries + representations.put(left,right); + } + return left + strOperator + right; + } + + // @TODO I'm unsure where this would be called. + @Override public String visitBinaryOperator(BinaryOperatorKind operator, String s, List list) + throws ExpressionVisitException, ODataApplicationException + { + String strOperator = BINARY_OPERATORS.get(operator); + throw new ODataApplicationException("Unsupported binary operation: " + operator.name(), + operator == BinaryOperatorKind.HAS ? + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode() : + HttpStatusCode.BAD_REQUEST.getStatusCode(), Locale.ENGLISH); + } + + @Override + public String visitUnaryOperator(UnaryOperatorKind operator, String operand) + throws ExpressionVisitException, ODataApplicationException { + switch (operator) { + case NOT: + return "NOT " + operand; + case MINUS: + return "-" + operand; + } + throw new ODataApplicationException("Wrong unary operator: " + operator, + HttpStatusCode.BAD_REQUEST.getStatusCode(), Locale.ENGLISH); + } + + @Override + public String visitMethodCall(MethodKind methodCall, List parameters) + throws ExpressionVisitException, ODataApplicationException { + if (parameters.isEmpty() && methodCall.equals(MethodKind.NOW)) { + return "CURRENT_DATE"; + } + String firsEntityParam = parameters.get(0); + switch (methodCall) { + case CONTAINS: + return firsEntityParam + " LIKE '%" + extractFromStringValue(parameters.get(1)) + "%'"; + case STARTSWITH: + return firsEntityParam + " LIKE '" + extractFromStringValue(parameters.get(1)) + "%'"; + case ENDSWITH: + return firsEntityParam + " LIKE '%" + extractFromStringValue(parameters.get(1)); + case DAY: + return "DAY(" + firsEntityParam + ")"; + case MONTH: + return "MONTH(" + firsEntityParam + ")"; + case YEAR: + return "YEAR(" + firsEntityParam + ")"; + } + throw new ODataApplicationException("Method call " + methodCall + " not implemented", + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH); + } + + @Override + public String visitLiteral(Literal literal) throws ExpressionVisitException, ODataApplicationException { + String literalAsString = literal.getText(); + if (literal.getType() == null) { + literalAsString = "NULL"; + } + + return literalAsString; + } + + @Override + public String visitMember(Member member) throws ExpressionVisitException, ODataApplicationException { + List resources = member.getResourcePath().getUriResourceParts(); + + UriResource first = resources.get(0); + + // TODO: Enum and ComplexType; joins + if (resources.size() == 1 && first instanceof UriResourcePrimitiveProperty) { + UriResourcePrimitiveProperty primitiveProperty = (UriResourcePrimitiveProperty) first; + return primitiveProperty.getProperty().getName(); + } else { + throw new ODataApplicationException("Only primitive properties are implemented in filter expressions", + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH); + } + } + + @Override + public String visitEnum(EdmEnumType type, List enumValues) + throws ExpressionVisitException, ODataApplicationException { + throw new ODataApplicationException("Enums are not implemented", HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), + Locale.ENGLISH); + } + + @Override + public String visitLambdaExpression(String lambdaFunction, String lambdaVariable, Expression expression) + throws ExpressionVisitException, ODataApplicationException { + throw new ODataApplicationException("Lambda expressions are not implemented", + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH); + } + + @Override + public String visitAlias(String aliasName) throws ExpressionVisitException, ODataApplicationException { + throw new ODataApplicationException("Aliases are not implemented", + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH); + } + + @Override + public String visitTypeLiteral(EdmType type) throws ExpressionVisitException, ODataApplicationException { + throw new ODataApplicationException("Type literals are not implemented", + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH); + } + + @Override + public String visitLambdaReference(String variableName) throws ExpressionVisitException, ODataApplicationException { + throw new ODataApplicationException("Lambda references are not implemented", + HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH); + } + + private String extractFromStringValue(String val) { + return val.substring(1, val.length() - 1); + } +} diff --git a/src/main/java/org/reso/service/data/meta/FieldInfo.java b/src/main/java/org/reso/service/data/meta/FieldInfo.java index c3dceac..2f5e367 100644 --- a/src/main/java/org/reso/service/data/meta/FieldInfo.java +++ b/src/main/java/org/reso/service/data/meta/FieldInfo.java @@ -41,7 +41,8 @@ public class FieldInfo public String getFieldName() { - return fieldName.toLowerCase(); + return fieldName; + //return fieldName.toLowerCase(); // For PostgreSQL. Not currently supported, but here if you want to play with it. } public String getODATAFieldName() { diff --git a/src/main/java/org/reso/service/data/meta/ResourceInfo.java b/src/main/java/org/reso/service/data/meta/ResourceInfo.java index b5c6f5e..a252a75 100644 --- a/src/main/java/org/reso/service/data/meta/ResourceInfo.java +++ b/src/main/java/org/reso/service/data/meta/ResourceInfo.java @@ -2,9 +2,14 @@ package org.reso.service.data.meta; import org.apache.olingo.commons.api.data.Entity; +import org.apache.olingo.commons.api.data.EntityCollection; import org.apache.olingo.commons.api.data.Property; import org.apache.olingo.commons.api.data.ValueType; +import org.apache.olingo.commons.api.edm.EdmEntitySet; import org.apache.olingo.commons.api.edm.FullQualifiedName; +import org.apache.olingo.server.api.ODataApplicationException; +import org.apache.olingo.server.api.uri.UriInfo; +import org.apache.olingo.server.api.uri.UriParameter; import org.reso.service.data.common.CommonDataProcessing; import org.reso.service.servlet.RESOservlet; import org.slf4j.Logger; @@ -12,6 +17,7 @@ import org.slf4j.LoggerFactory; import java.sql.*; import java.util.ArrayList; +import java.util.List; public class ResourceInfo { @@ -52,6 +58,8 @@ public class ResourceInfo return null; } + public Boolean useCustomDatasource() { return false; } + public FullQualifiedName getFqn(String namespace) { if (this.fqn==null) @@ -90,4 +98,14 @@ public class ResourceInfo this.primaryKeyName = primaryKey; } + + public Entity getData(EdmEntitySet edmEntitySet, List keyPredicates) + { + return null; + } + + public EntityCollection getData(EdmEntitySet edmEntitySet, UriInfo uriInfo, boolean isCount) throws ODataApplicationException + { + return null; + } } diff --git a/src/main/java/org/reso/service/servlet/RESOservlet.java b/src/main/java/org/reso/service/servlet/RESOservlet.java index 7f2172f..008c362 100644 --- a/src/main/java/org/reso/service/servlet/RESOservlet.java +++ b/src/main/java/org/reso/service/servlet/RESOservlet.java @@ -9,6 +9,7 @@ import org.apache.olingo.server.api.ServiceMetadata; import org.reso.service.data.GenericEntityCollectionProcessor; import org.reso.service.data.GenericEntityProcessor; import org.reso.service.data.definition.LookupDefinition; +import org.reso.service.data.definition.custom.FieldDefinition; import org.reso.service.data.meta.ResourceInfo; import org.reso.service.edmprovider.RESOedmProvider; import org.reso.service.security.providers.BasicAuthProvider; @@ -81,23 +82,13 @@ public class RESOservlet extends HttpServlet } else LOG.info("DB String unknown form: "+dbConnString); } - LOG.info("DB String empty"); try { Class.forName(dbDriverStr).newInstance(); LOG.debug("looking to connect to " + dbConnString); - if (this.dbType.equals("mysql")) - { - - connect = DriverManager - .getConnection(dbConnString); - } - else if (this.dbType.equals("postgresql")) - { - connect = DriverManager - .getConnection(dbConnString,dbUser,dbPwd); - } + connect = DriverManager + .getConnection(dbConnString,dbUser,dbPwd); } catch (Exception e) { @@ -116,6 +107,11 @@ public class RESOservlet extends HttpServlet ArrayList resources = new ArrayList<>(); + // We are going to use a custom field definition to query Fields + FieldDefinition fieldDefinition = new FieldDefinition(); + resources.add(fieldDefinition); + fieldDefinition.addResources(resources); + // Get all classes with constructors with 0 parameters. LookupDefinition should not work. try { @@ -133,10 +129,17 @@ public class RESOservlet extends HttpServlet { ctor.setAccessible(true); ResourceInfo resource = (ResourceInfo)ctor.newInstance(); - resources.add(resource); - resourceLookup.put(resource.getResourceName(), resource); - resource.findPrimaryKey(this.connect); + try + { + resource.findPrimaryKey(this.connect); + resources.add(resource); + resourceLookup.put(resource.getResourceName(), resource); + } + catch (Exception e) + { + LOG.error("Error with: "+resource.getResourceName()+" - "+e.getMessage()); + } } } } From 77b723f95a92417f9438d8153344953bd0560565 Mon Sep 17 00:00:00 2001 From: michaelpede Date: Wed, 18 Aug 2021 14:45:31 -0700 Subject: [PATCH 2/2] Progress importing Certification metadata report JSON also, getting rid of Postgres modifications --- .env | 1 + .gitignore | 2 + pom.xml | 5 + .../GenericEntityCollectionProcessor.java | 2 +- .../service/data/GenericEntityProcessor.java | 2 +- .../{custom => }/FieldDefinition.java | 9 +- .../service/data/meta/DefinitionBuilder.java | 170 ++++++++++++++++++ .../reso/service/data/meta/ResourceInfo.java | 4 +- .../org/reso/service/servlet/RESOservlet.java | 37 +++- 9 files changed, 214 insertions(+), 18 deletions(-) rename src/main/java/org/reso/service/data/definition/{custom => }/FieldDefinition.java (95%) create mode 100644 src/main/java/org/reso/service/data/meta/DefinitionBuilder.java diff --git a/.env b/.env index d9c3320..df491d7 100644 --- a/.env +++ b/.env @@ -6,3 +6,4 @@ SQL_DB_DRIVER=com.mysql.cj.jdbc.Driver #SQL_DB_DRIVER=org.postgresql.Driver SQL_CONNECTION_STR=jdbc:mysql://192.168.1.71/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4&user=mpede&password=password #SQL_CONNECTION_STR=jdbc:postgresql://192.168.1.71:5432/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4 +CERT_REPORT_FILENAME=RESODataDictionary-1.7.metadata-report.json \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5265e28..af21487 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ hs_err_pid* /sql /web /build +/.idea # Project Genereated files .pmd @@ -38,6 +39,7 @@ nb-configuration.xml .externalToolBuilders maven-eclipse.xml dependency-reduced-pom.xml +src/main/java/org/reso/service/data/definition/custom/*.java src/main/java/org/reso/service/data/definition/ContactListingNotesDefinition.java src/main/java/org/reso/service/data/definition/ContactListingsDefinition.java src/main/java/org/reso/service/data/definition/ContactsDefinition.java diff --git a/pom.xml b/pom.xml index d1c613f..b86acc4 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,11 @@ 2.11.4 + + com.google.code.gson + gson + 2.8.7 + diff --git a/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java b/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java index aa6faf1..c6acd2e 100644 --- a/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java +++ b/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java @@ -260,7 +260,7 @@ public class GenericEntityCollectionProcessor implements EntityCollectionProcess if (uriResource instanceof UriResourcePrimitiveProperty) { EdmProperty edmProperty = ((UriResourcePrimitiveProperty) uriResource).getProperty(); - final String sortPropertyName = edmProperty.getName().toLowerCase(); + final String sortPropertyName = edmProperty.getName(); // .toLowerCase(); queryString = queryString + " ORDER BY "+sortPropertyName; if(orderByItem.isDescending()) { diff --git a/src/main/java/org/reso/service/data/GenericEntityProcessor.java b/src/main/java/org/reso/service/data/GenericEntityProcessor.java index abc6f6e..95d8c66 100644 --- a/src/main/java/org/reso/service/data/GenericEntityProcessor.java +++ b/src/main/java/org/reso/service/data/GenericEntityProcessor.java @@ -111,7 +111,7 @@ public class GenericEntityProcessor implements EntityProcessor for (final UriParameter key : keyPredicates) { // key - String keyName = key.getName().toLowerCase(); + String keyName = key.getName(); // .toLowerCase(); String keyValue = key.getText(); if (sqlCriteria==null) { diff --git a/src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java b/src/main/java/org/reso/service/data/definition/FieldDefinition.java similarity index 95% rename from src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java rename to src/main/java/org/reso/service/data/definition/FieldDefinition.java index 2ba0405..519b1d8 100644 --- a/src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java +++ b/src/main/java/org/reso/service/data/definition/FieldDefinition.java @@ -1,4 +1,4 @@ -package org.reso.service.data.definition.custom; +package org.reso.service.data.definition; import org.apache.olingo.commons.api.data.Entity; import org.apache.olingo.commons.api.data.EntityCollection; @@ -6,8 +6,6 @@ import org.apache.olingo.commons.api.data.Property; import org.apache.olingo.commons.api.data.ValueType; import org.apache.olingo.commons.api.edm.EdmEntitySet; import org.apache.olingo.commons.api.edm.EdmPrimitiveTypeKind; -import org.apache.olingo.commons.api.edm.provider.CsdlAnnotation; -import org.apache.olingo.commons.api.edm.provider.CsdlEntitySet; import org.apache.olingo.server.api.ODataApplicationException; import org.apache.olingo.server.api.uri.UriInfo; import org.apache.olingo.server.api.uri.UriParameter; @@ -16,12 +14,10 @@ import org.apache.olingo.server.api.uri.queryoption.expression.ExpressionVisitEx import org.reso.service.data.common.CommonDataProcessing; import org.reso.service.data.meta.BreakdownOfFilterExpressionVisitor; import org.reso.service.data.meta.FieldInfo; -import org.reso.service.data.meta.MySQLFilterExpressionVisitor; import org.reso.service.data.meta.ResourceInfo; import java.sql.ResultSet; import java.sql.Statement; -import java.sql.Timestamp; import java.util.*; public class FieldDefinition extends ResourceInfo @@ -36,6 +32,7 @@ public class FieldDefinition extends ResourceInfo this.tableName = "field"; // Never used this.resourcesName = "Field"; this.resourceName = "Field"; + this.primaryKeyName = "FieldKey"; } public ArrayList getFieldList() @@ -95,7 +92,7 @@ public class FieldDefinition extends ResourceInfo for (final UriParameter key : keyPredicates) { // key - String keyName = key.getName().toLowerCase(); + String keyName = key.getName(); // .toLowerCase(); String keyValue = key.getText(); if (sqlCriteria==null) { diff --git a/src/main/java/org/reso/service/data/meta/DefinitionBuilder.java b/src/main/java/org/reso/service/data/meta/DefinitionBuilder.java new file mode 100644 index 0000000..119be9b --- /dev/null +++ b/src/main/java/org/reso/service/data/meta/DefinitionBuilder.java @@ -0,0 +1,170 @@ +package org.reso.service.data.meta; + +import com.google.gson.stream.JsonReader; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +class AnnotationObject +{ + private static Map FIELD_PROPERTIES = Stream.of( + new AbstractMap.SimpleEntry<>("term", String.class), + new AbstractMap.SimpleEntry<>("value", String.class) ) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +} + +class FieldObject +{ + private static Map FIELD_PROPERTIES = Stream.of( + new AbstractMap.SimpleEntry<>("resourceName", String.class), + new AbstractMap.SimpleEntry<>("fieldName", String.class), + new AbstractMap.SimpleEntry<>("type", String.class), + new AbstractMap.SimpleEntry<>("nullable", Boolean.class), + new AbstractMap.SimpleEntry<>("scale", Number.class), + new AbstractMap.SimpleEntry<>("precision", Number.class), + new AbstractMap.SimpleEntry<>("isCollection", Boolean.class), + new AbstractMap.SimpleEntry<>("unicode", Boolean.class) ) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + private JsonReader reader; + private HashMap properties = new HashMap<>(); + + public FieldObject(JsonReader reader) + { + this.reader = reader; + this.readObject(); + } + + private void readObject() + { + try + { + reader.beginObject(); + + while (reader.hasNext()) { + + String name = reader.nextName(); + + if (FIELD_PROPERTIES.containsKey(name)) + { + Object classType = FIELD_PROPERTIES.get(name); + + if (classType.equals(String.class)) + { + properties.put(name,reader.nextString() ); + } + else if (classType.equals(Boolean.class)) + { + properties.put(name,reader.nextBoolean() ); + } + else if (classType.equals(Number.class)) + { + properties.put(name,reader.nextLong() ); + } + } else if (name.equals("annotations")) { + // read array + reader.beginArray(); + + while (reader.hasNext()) { + reader.skipValue(); + //fields.add( this.readField() ); + } + reader.endArray(); + + } else { + reader.skipValue(); //avoid some unhandle events + } + } + + reader.endObject(); + } + catch (IOException e) + { + e.printStackTrace(); + } + } + + +} + +public class DefinitionBuilder +{ + private static Map HEADER_FIELDS = Stream.of( + new AbstractMap.SimpleEntry<>("description", true), + new AbstractMap.SimpleEntry<>("generatedOn", true), + new AbstractMap.SimpleEntry<>("version", true)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + private String fileName; + private JsonReader reader; + + public DefinitionBuilder(String fileName) + { + this.fileName = fileName; + this.openFile(); + } + + public void openFile() + { + try + { + reader = new JsonReader(new FileReader(fileName)); + } + catch (FileNotFoundException e) + { + e.printStackTrace(); + } + } + + private FieldObject readField() + { + FieldObject fo = new FieldObject(reader); + return fo; + } + + public List readResources() + { + List resources = new ArrayList<>(); + ArrayList fields = new ArrayList(); + + try + { + reader.beginObject(); + + while (reader.hasNext()) { + + String name = reader.nextName(); + + if (HEADER_FIELDS.containsKey(name)) + { + String headerValue = reader.nextString(); + } else if (name.equals("fields")) { + + // read array + reader.beginArray(); + + while (reader.hasNext()) { + fields.add( this.readField() ); + } + + reader.endArray(); + + } else { + reader.skipValue(); //avoid some unhandle events + } + } + + reader.endObject(); + } + catch (IOException e) + { + e.printStackTrace(); + } + + return resources; + } +} diff --git a/src/main/java/org/reso/service/data/meta/ResourceInfo.java b/src/main/java/org/reso/service/data/meta/ResourceInfo.java index a252a75..f8e3c3b 100644 --- a/src/main/java/org/reso/service/data/meta/ResourceInfo.java +++ b/src/main/java/org/reso/service/data/meta/ResourceInfo.java @@ -79,10 +79,10 @@ public class ResourceInfo String pkColumnName = pkColumns.getString("COLUMN_NAME"); Integer pkPosition = pkColumns.getInt("KEY_SEQ"); LOG.debug(""+pkColumnName+" is the "+pkPosition+". column of the primary key of the table "+tableName); - primaryKey = pkColumnName.toLowerCase(); + primaryKey = pkColumnName; //.toLowerCase(); // lowercase only needed for PostgreSQL } - String[] splitKey = primaryKey.split("numeric"); + String[] splitKey = primaryKey.split("Numeric"); if (splitKey.length>=1) primaryKey = splitKey[0]; diff --git a/src/main/java/org/reso/service/servlet/RESOservlet.java b/src/main/java/org/reso/service/servlet/RESOservlet.java index 008c362..5565c9e 100644 --- a/src/main/java/org/reso/service/servlet/RESOservlet.java +++ b/src/main/java/org/reso/service/servlet/RESOservlet.java @@ -9,10 +9,10 @@ import org.apache.olingo.server.api.ServiceMetadata; import org.reso.service.data.GenericEntityCollectionProcessor; import org.reso.service.data.GenericEntityProcessor; import org.reso.service.data.definition.LookupDefinition; -import org.reso.service.data.definition.custom.FieldDefinition; +import org.reso.service.data.meta.DefinitionBuilder; +import org.reso.service.data.definition.FieldDefinition; import org.reso.service.data.meta.ResourceInfo; import org.reso.service.edmprovider.RESOedmProvider; -import org.reso.service.security.providers.BasicAuthProvider; import org.reso.service.security.Validator; import org.reso.service.security.providers.BearerAuthProvider; import org.reso.service.servlet.util.ClassLoader; @@ -24,10 +24,8 @@ import java.io.*; import java.lang.reflect.Constructor; import java.sql.Connection; import java.sql.DriverManager; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; +import java.sql.SQLException; +import java.util.*; import javax.servlet.ServletException; import javax.servlet.http.*; @@ -72,6 +70,9 @@ public class RESOservlet extends HttpServlet String dbConnString = env.get("SQL_CONNECTION_STR"); String dbDriverStr = env.get("SQL_DB_DRIVER"); + + String definitionFile = env.get("CERT_REPORT_FILENAME"); + if (dbConnString!=null) { String[] dbConnSplitStr = dbConnString.split(":"); @@ -111,11 +112,21 @@ public class RESOservlet extends HttpServlet FieldDefinition fieldDefinition = new FieldDefinition(); resources.add(fieldDefinition); fieldDefinition.addResources(resources); + resourceLookup.put(fieldDefinition.getResourceName(), fieldDefinition); + + + // If there is a Certification metadata report file, import it for class definitions. + + if (definitionFile!=null && false) + { + DefinitionBuilder definitionBuilder = new DefinitionBuilder(definitionFile); + List loadedResources = definitionBuilder.readResources(); + } // Get all classes with constructors with 0 parameters. LookupDefinition should not work. try { - Class[] classList = ClassLoader.getClasses("org.reso.service.data.definition"); + Class[] classList = ClassLoader.getClasses("org.reso.service.data.definition.custom"); for (Class classProto: classList) { Constructor ctor = null; @@ -148,7 +159,17 @@ public class RESOservlet extends HttpServlet LOG.error(e.getMessage()); } -// ResourceInfo defn = new LookupDefinition(); + ResourceInfo defn = new LookupDefinition(); + try + { + defn.findPrimaryKey(this.connect); + resources.add(defn); + resourceLookup.put(defn.getResourceName(), defn); + } + catch (Exception e) + { + LOG.error(e.getMessage()); + } ServiceMetadata edm = odata.createServiceMetadata(edmProvider, new ArrayList());