USRealEstate
/
reso-web-api-reference-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add DDL for Field table. 

Also, a new build process that uses docker to build parts that fail on Macs
This commit is contained in:
michaelpede 2021-08-17 12:26:35 -07:00
parent d726f30fdd
commit 0833c90424
11 changed files with 513 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.env
View File

@ -2,7 +2,7 @@ COMPOSE__FILE=docker-compose.yml:./optional/docker-db-compose.yml
SQL_HOST=192.168.1.71
SQL_USER=mpede
SQL_PASSWORD=password
#SQL_DB_DRIVER=com.mysql.cj.jdbc.Driver
SQL_DB_DRIVER=org.postgresql.Driver
#SQL_CONNECTION_STR=jdbc:mysql://192.168.1.71/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4&user=mpede&password=password
SQL_CONNECTION_STR=jdbc:postgresql://192.168.1.71:5432/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4
SQL_DB_DRIVER=com.mysql.cj.jdbc.Driver
#SQL_DB_DRIVER=org.postgresql.Driver
SQL_CONNECTION_STR=jdbc:mysql://192.168.1.71/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4&user=mpede&password=password
#SQL_CONNECTION_STR=jdbc:postgresql://192.168.1.71:5432/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4

36
build.sh
View File

@ -5,40 +5,8 @@ HOME_DIR=`dirname ${REAL_VAR0}`
TEMP_DIR="${HOME_DIR}/temp"
SQL_DIR="${HOME_DIR}/sql"
# Ensure we have directories set up
[ -d "${TEMP_DIR}" ] && echo "temp directory found." || mkdir ${TEMP_DIR}
[ -d "${SQL_DIR}" ] && echo "sql directory found." || mkdir ${SQL_DIR}
if [ -z "${SQL_HOST}" ]
then
# Get the Web API Commander, needed to generate the test database
if ! wget https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar -O temp/web-api-commander.jar
then
echo "WGET not installed. trying CURL."
if ! curl -L https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar --output temp/web-api-commander.jar
then
echo "CURL not installed. Exiting build."
exit
fi
fi
java -jar temp/web-api-commander.jar --generateReferenceDDL --useKeyNumeric > sql/reso-reference-ddl-dd-1.7.numeric-keys.sql
# The following lines should be independent of the SQL logic.
java -jar temp/web-api-commander.jar --generateResourceInfoModels
mv ResourceInfoModels/* src/main/java/org/reso/service/data/definition/
else
COMPOSE_FILE="docker-compose.yml"
fi
if ! mvn compile
then
echo "Maven could not be found."
exit
else
mvn package
fi
docker build -t reso-builder -f docker/docker-builder .
docker run --name builder --mount type=bind,source="${HOME_DIR}",target=/usr/src/app -t reso-builder
if ! docker-compose build
then

7
docker/docker-builder Normal file
View File

@ -0,0 +1,7 @@
FROM ubuntu:20.10
WORKDIR /usr/src/app
RUN apt-get update && apt-get install -y wget curl pip maven docker-compose
CMD ./docker/scripts/build.sh

42
docker/scripts/build.sh Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
REAL_VAR0=`readlink -f $0`
HOME_DIR=`dirname ${REAL_VAR0}`
TEMP_DIR="${HOME_DIR}/temp"
SQL_DIR="${HOME_DIR}/sql"
# Ensure we have directories set up
[ -d "${TEMP_DIR}" ] && echo "temp directory found." || mkdir ${TEMP_DIR}
[ -d "${SQL_DIR}" ] && echo "sql directory found." || mkdir ${SQL_DIR}
if [ -z "${SQL_HOST}" ]
then
# Get the Web API Commander, needed to generate the test database
if ! wget https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar -O temp/web-api-commander.jar
then
echo "WGET not installed. trying CURL."
if ! curl -L https://github.com/RESOStandards/web-api-commander/releases/download/current-version/web-api-commander.jar --output temp/web-api-commander.jar
then
echo "CURL not installed. Exiting build."
exit
fi
fi
java -jar temp/web-api-commander.jar --generateReferenceDDL --useKeyNumeric > sql/reso-reference-ddl-dd-1.7.numeric-keys.sql
# The following lines should be independent of the SQL logic.
java -jar temp/web-api-commander.jar --generateResourceInfoModels
mv ResourceInfoModels/* src/main/java/org/reso/service/data/definition/
else
COMPOSE_FILE="docker-compose.yml"
fi
if ! mvn compile
then
echo "Maven could not be found."
exit
else
mvn package
fi

11
src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java
View File

@ -85,7 +85,16 @@ public class GenericEntityCollectionProcessor implements EntityCollectionProcess
// 2nd: fetch the data from backend for this requested EntitySetName
// it has to be delivered as EntitySet object
EntityCollection entitySet = getData(edmEntitySet, uriInfo, isCount, resource);
EntityCollection entitySet;
if (resource.useCustomDatasource() )
{
entitySet = resource.getData(edmEntitySet, uriInfo, isCount);
}
else
{
entitySet = getData(edmEntitySet, uriInfo, isCount, resource);
}
// 3rd: create a serializer based on the requested format (json)
try

10
src/main/java/org/reso/service/data/GenericEntityProcessor.java
View File

@ -65,7 +65,15 @@ public class GenericEntityProcessor implements EntityProcessor
// 2. retrieve the data from backend
List<UriParameter> keyPredicates = uriResourceEntitySet.getKeyPredicates();
Entity entity = getData(edmEntitySet, keyPredicates, resource);
Entity entity;
if (resource.useCustomDatasource() )
{
entity = resource.getData(edmEntitySet, keyPredicates);
}
else
{
entity = getData(edmEntitySet, keyPredicates, resource);
}
// 3. serialize
EdmEntityType entityType = edmEntitySet.getEntityType();

221
src/main/java/org/reso/service/data/definition/custom/FieldDefinition.java Normal file
View File

@ -0,0 +1,221 @@
package org.reso.service.data.definition.custom;
import org.apache.olingo.commons.api.data.Entity;
import org.apache.olingo.commons.api.data.EntityCollection;
import org.apache.olingo.commons.api.data.Property;
import org.apache.olingo.commons.api.data.ValueType;
import org.apache.olingo.commons.api.edm.EdmEntitySet;
import org.apache.olingo.commons.api.edm.EdmPrimitiveTypeKind;
import org.apache.olingo.commons.api.edm.provider.CsdlAnnotation;
import org.apache.olingo.commons.api.edm.provider.CsdlEntitySet;
import org.apache.olingo.server.api.ODataApplicationException;
import org.apache.olingo.server.api.uri.UriInfo;
import org.apache.olingo.server.api.uri.UriParameter;
import org.apache.olingo.server.api.uri.queryoption.FilterOption;
import org.apache.olingo.server.api.uri.queryoption.expression.ExpressionVisitException;
import org.reso.service.data.common.CommonDataProcessing;
import org.reso.service.data.meta.BreakdownOfFilterExpressionVisitor;
import org.reso.service.data.meta.FieldInfo;
import org.reso.service.data.meta.MySQLFilterExpressionVisitor;
import org.reso.service.data.meta.ResourceInfo;
import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.*;
public class FieldDefinition extends ResourceInfo
{
private static String STANDARD_NAME = "RESO.OData.Metadata.StandardName";
private static ArrayList<FieldInfo> fieldList = null;
private ArrayList<ResourceInfo> resources;
public FieldDefinition()
{
this.tableName = "field"; // Never used
this.resourcesName = "Field";
this.resourceName = "Field";
}
public ArrayList<FieldInfo> getFieldList()
{
return FieldDefinition.getStaticFieldList();
}
public static ArrayList<FieldInfo> getStaticFieldList()
{
if (null!= FieldDefinition.fieldList)
{
return FieldDefinition.fieldList;
}
ArrayList<FieldInfo> list = new ArrayList<FieldInfo>();
FieldDefinition.fieldList = list;
FieldInfo fieldInfo = null;
fieldInfo = new FieldInfo("FieldKey", EdmPrimitiveTypeKind.String.getFullQualifiedName());
fieldInfo.addAnnotation("Field Key Field", "RESO.OData.Metadata.DisplayName");
list.add(fieldInfo);
fieldInfo = new FieldInfo("ResourceName", EdmPrimitiveTypeKind.String.getFullQualifiedName());
list.add(fieldInfo);
fieldInfo = new FieldInfo("FieldName", EdmPrimitiveTypeKind.String.getFullQualifiedName());
list.add(fieldInfo);
fieldInfo = new FieldInfo("DisplayName", EdmPrimitiveTypeKind.String.getFullQualifiedName());
list.add(fieldInfo);
fieldInfo = new FieldInfo("ModificationTimestamp", EdmPrimitiveTypeKind.DateTimeOffset.getFullQualifiedName());
list.add(fieldInfo);
return FieldDefinition.fieldList;
}
public Boolean useCustomDatasource() { return true; }
public Entity getData(EdmEntitySet edmEntitySet, List<UriParameter> keyPredicates)
{
ArrayList<FieldInfo> fields = this.getFieldList();
Entity product = null;
Map<String, String> properties = System.getenv();
try {
String sqlCriteria = null;
// Statements allow to issue SQL queries to the database
Statement statement = null;
// Result set get the result of the SQL query
String queryString = null;
for (final UriParameter key : keyPredicates)
{
// key
String keyName = key.getName().toLowerCase();
String keyValue = key.getText();
if (sqlCriteria==null)
{
sqlCriteria = keyName + " = " + keyValue;
}
else
{
sqlCriteria = sqlCriteria + " and " + keyName + " = " + keyValue;
}
}
queryString = "select * from " + this.getTableName();
if (null!=sqlCriteria && sqlCriteria.length()>0)
{
queryString = queryString + " WHERE " + sqlCriteria;
}
LOG.info("SQL Query: "+queryString);
ResultSet resultSet = statement.executeQuery(queryString);
String primaryFieldName = this.getPrimaryKeyName();
// add the lookups from the database.
while (resultSet.next())
{
Entity ent = CommonDataProcessing.getEntityFromRow(resultSet, this, null);
product = ent;
}
statement.close();
} catch (Exception e) {
LOG.error("Server Error occurred in reading "+this.getResourceName(), e);
return product;
}
return product;
}
public EntityCollection getData(EdmEntitySet edmEntitySet, UriInfo uriInfo, boolean isCount) throws ODataApplicationException
{
EntityCollection entCollection = new EntityCollection();
List<Entity> productList = entCollection.getEntities();
FilterOption filter = uriInfo.getFilterOption();
BreakdownOfFilterExpressionVisitor customExpression = new BreakdownOfFilterExpressionVisitor(this);
try
{
String criteria = filter.getExpression().accept(customExpression);
}
catch (ExpressionVisitException e)
{
LOG.error("Server Error occurred in reading "+this.getResourceName(), e);
return entCollection;
}
HashMap<String,String> reps = customExpression.getRepresentations();
String resourceName = reps.remove("ResourceName");
resourceName = resourceName.substring(1,resourceName.length()-1);
ResourceInfo res = null;
for (ResourceInfo defn :resources)
{
if (resourceName.equals(defn.getResourcesName()))
{
res = defn;
break;
}
}
ArrayList<FieldInfo> resourceFieldList = res.getFieldList();
for (FieldInfo field: resourceFieldList)
{
HashMap<String,Object> entityValues = new HashMap<>();
entityValues.put("FieldKey", field.getFieldName());
entityValues.put("FieldName", field.getFieldName());
entityValues.put("ResourceName", resourceName);
entityValues.put("DisplayName", field.getFieldName());
Date date = new Date();
entityValues.put("ModificationTimestamp", date);
boolean match = true;
for (String key: reps.keySet() )
{
String toMatch = reps.get(key);
toMatch = toMatch.substring(1,toMatch.length()-1);
String value = entityValues.get(key).toString();
if (!toMatch.equals(value))
{
match = false;
break;
}
}
if (match)
{
Entity ent = new Entity();
ent.addProperty(new Property(null, "FieldKey", ValueType.PRIMITIVE, field.getFieldName()));
ent.addProperty(new Property(null, "FieldName", ValueType.PRIMITIVE, field.getFieldName()));
ent.addProperty(new Property(null, "ResourceName", ValueType.PRIMITIVE, resourceName));
String displayName = field.getFieldName();
ent.addProperty(new Property(null, "DisplayName", ValueType.PRIMITIVE, displayName));
ent.addProperty(new Property(null, "ModificationTimestamp", ValueType.PRIMITIVE, date ));
productList.add(ent);
}
}
return entCollection;
}
public void addResources(ArrayList<ResourceInfo> resources)
{
this.resources = resources;
}
}

179
src/main/java/org/reso/service/data/meta/BreakdownOfFilterExpressionVisitor.java Normal file
View File

@ -0,0 +1,179 @@
package org.reso.service.data.meta;
import org.apache.olingo.commons.api.edm.EdmEnumType;
import org.apache.olingo.commons.api.edm.EdmPrimitiveTypeKind;
import org.apache.olingo.commons.api.edm.EdmType;
import org.apache.olingo.commons.api.http.HttpStatusCode;
import org.apache.olingo.server.api.ODataApplicationException;
import org.apache.olingo.server.api.uri.UriResource;
import org.apache.olingo.server.api.uri.UriResourcePrimitiveProperty;
import org.apache.olingo.server.api.uri.queryoption.expression.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class BreakdownOfFilterExpressionVisitor implements ExpressionVisitor<String>
{
private static final Logger LOG = LoggerFactory.getLogger(MySQLFilterExpressionVisitor.class);
private static final Map<BinaryOperatorKind, String> BINARY_OPERATORS = new HashMap<BinaryOperatorKind, String>() {{
put(BinaryOperatorKind.ADD, " + ");
put(BinaryOperatorKind.AND, " AND ");
put(BinaryOperatorKind.DIV, " / ");
put(BinaryOperatorKind.EQ, " = ");
put(BinaryOperatorKind.GE, " >= ");
put(BinaryOperatorKind.GT, " > ");
put(BinaryOperatorKind.LE, " <= ");
put(BinaryOperatorKind.LT, " < ");
put(BinaryOperatorKind.MOD, " % ");
put(BinaryOperatorKind.MUL, " * ");
put(BinaryOperatorKind.NE, " <> ");
put(BinaryOperatorKind.OR, " OR ");
put(BinaryOperatorKind.SUB, " - ");
}};
private HashMap<String,String> representations = new HashMap<>();
private ResourceInfo resourceInfo;
public BreakdownOfFilterExpressionVisitor(ResourceInfo resourceInfo) {
this.resourceInfo = resourceInfo;
}
public HashMap<String,String> getRepresentations()
{
return representations;
}
@Override
public String visitBinaryOperator(BinaryOperatorKind operator, String left, String right)
throws ExpressionVisitException, ODataApplicationException
{
String strOperator = BINARY_OPERATORS.get(operator);
if (strOperator == null) {
throw new ODataApplicationException("Unsupported binary operation: " + operator.name(),
operator == BinaryOperatorKind.HAS ?
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode() :
HttpStatusCode.BAD_REQUEST.getStatusCode(), Locale.ENGLISH);
}
// ResourceName eq 'Property' becomes (=, ResourceName, Property)
if ( "=".equals(strOperator.trim() ) ) // We only want the equals operator from the filter. Assume all are AND ops for now if there are multiples.
{ // @TODO: Make this filter more robust for queries
representations.put(left,right);
}
return left + strOperator + right;
}
// @TODO I'm unsure where this would be called.
@Override public String visitBinaryOperator(BinaryOperatorKind operator, String s, List<String> list)
throws ExpressionVisitException, ODataApplicationException
{
String strOperator = BINARY_OPERATORS.get(operator);
throw new ODataApplicationException("Unsupported binary operation: " + operator.name(),
operator == BinaryOperatorKind.HAS ?
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode() :
HttpStatusCode.BAD_REQUEST.getStatusCode(), Locale.ENGLISH);
}
@Override
public String visitUnaryOperator(UnaryOperatorKind operator, String operand)
throws ExpressionVisitException, ODataApplicationException {
switch (operator) {
case NOT:
return "NOT " + operand;
case MINUS:
return "-" + operand;
}
throw new ODataApplicationException("Wrong unary operator: " + operator,
HttpStatusCode.BAD_REQUEST.getStatusCode(), Locale.ENGLISH);
}
@Override
public String visitMethodCall(MethodKind methodCall, List<String> parameters)
throws ExpressionVisitException, ODataApplicationException {
if (parameters.isEmpty() && methodCall.equals(MethodKind.NOW)) {
return "CURRENT_DATE";
}
String firsEntityParam = parameters.get(0);
switch (methodCall) {
case CONTAINS:
return firsEntityParam + " LIKE '%" + extractFromStringValue(parameters.get(1)) + "%'";
case STARTSWITH:
return firsEntityParam + " LIKE '" + extractFromStringValue(parameters.get(1)) + "%'";
case ENDSWITH:
return firsEntityParam + " LIKE '%" + extractFromStringValue(parameters.get(1));
case DAY:
return "DAY(" + firsEntityParam + ")";
case MONTH:
return "MONTH(" + firsEntityParam + ")";
case YEAR:
return "YEAR(" + firsEntityParam + ")";
}
throw new ODataApplicationException("Method call " + methodCall + " not implemented",
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH);
}
@Override
public String visitLiteral(Literal literal) throws ExpressionVisitException, ODataApplicationException {
String literalAsString = literal.getText();
if (literal.getType() == null) {
literalAsString = "NULL";
}
return literalAsString;
}
@Override
public String visitMember(Member member) throws ExpressionVisitException, ODataApplicationException {
List<UriResource> resources = member.getResourcePath().getUriResourceParts();
UriResource first = resources.get(0);
// TODO: Enum and ComplexType; joins
if (resources.size() == 1 && first instanceof UriResourcePrimitiveProperty) {
UriResourcePrimitiveProperty primitiveProperty = (UriResourcePrimitiveProperty) first;
return primitiveProperty.getProperty().getName();
} else {
throw new ODataApplicationException("Only primitive properties are implemented in filter expressions",
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH);
}
}
@Override
public String visitEnum(EdmEnumType type, List<String> enumValues)
throws ExpressionVisitException, ODataApplicationException {
throw new ODataApplicationException("Enums are not implemented", HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(),
Locale.ENGLISH);
}
@Override
public String visitLambdaExpression(String lambdaFunction, String lambdaVariable, Expression expression)
throws ExpressionVisitException, ODataApplicationException {
throw new ODataApplicationException("Lambda expressions are not implemented",
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH);
}
@Override
public String visitAlias(String aliasName) throws ExpressionVisitException, ODataApplicationException {
throw new ODataApplicationException("Aliases are not implemented",
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH);
}
@Override
public String visitTypeLiteral(EdmType type) throws ExpressionVisitException, ODataApplicationException {
throw new ODataApplicationException("Type literals are not implemented",
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH);
}
@Override
public String visitLambdaReference(String variableName) throws ExpressionVisitException, ODataApplicationException {
throw new ODataApplicationException("Lambda references are not implemented",
HttpStatusCode.NOT_IMPLEMENTED.getStatusCode(), Locale.ENGLISH);
}
private String extractFromStringValue(String val) {
return val.substring(1, val.length() - 1);
}
}

3
src/main/java/org/reso/service/data/meta/FieldInfo.java
View File

@ -41,7 +41,8 @@ public class FieldInfo
public String getFieldName()
{
return fieldName.toLowerCase();
return fieldName;
//return fieldName.toLowerCase(); // For PostgreSQL. Not currently supported, but here if you want to play with it.
}
public String getODATAFieldName()
{

18
src/main/java/org/reso/service/data/meta/ResourceInfo.java
View File

@ -2,9 +2,14 @@ package org.reso.service.data.meta;
import org.apache.olingo.commons.api.data.Entity;
import org.apache.olingo.commons.api.data.EntityCollection;
import org.apache.olingo.commons.api.data.Property;
import org.apache.olingo.commons.api.data.ValueType;
import org.apache.olingo.commons.api.edm.EdmEntitySet;
import org.apache.olingo.commons.api.edm.FullQualifiedName;
import org.apache.olingo.server.api.ODataApplicationException;
import org.apache.olingo.server.api.uri.UriInfo;
import org.apache.olingo.server.api.uri.UriParameter;
import org.reso.service.data.common.CommonDataProcessing;
import org.reso.service.servlet.RESOservlet;
import org.slf4j.Logger;
@ -12,6 +17,7 @@ import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
public class ResourceInfo
{
@ -52,6 +58,8 @@ public class ResourceInfo
return null;
}
public Boolean useCustomDatasource() { return false; }
public FullQualifiedName getFqn(String namespace)
{
if (this.fqn==null)
@ -90,4 +98,14 @@ public class ResourceInfo
this.primaryKeyName = primaryKey;
}
public Entity getData(EdmEntitySet edmEntitySet, List<UriParameter> keyPredicates)
{
return null;
}
public EntityCollection getData(EdmEntitySet edmEntitySet, UriInfo uriInfo, boolean isCount) throws ODataApplicationException
{
return null;
}
}

27
src/main/java/org/reso/service/servlet/RESOservlet.java
View File

@ -9,6 +9,7 @@ import org.apache.olingo.server.api.ServiceMetadata;
import org.reso.service.data.GenericEntityCollectionProcessor;
import org.reso.service.data.GenericEntityProcessor;
import org.reso.service.data.definition.LookupDefinition;
import org.reso.service.data.definition.custom.FieldDefinition;
import org.reso.service.data.meta.ResourceInfo;
import org.reso.service.edmprovider.RESOedmProvider;
import org.reso.service.security.providers.BasicAuthProvider;
@ -81,23 +82,13 @@ public class RESOservlet extends HttpServlet
}
else LOG.info("DB String unknown form: "+dbConnString);
}
LOG.info("DB String empty");
try {
Class.forName(dbDriverStr).newInstance();
LOG.debug("looking to connect to " + dbConnString);
if (this.dbType.equals("mysql"))
{
connect = DriverManager
.getConnection(dbConnString);
}
else if (this.dbType.equals("postgresql"))
{
connect = DriverManager
.getConnection(dbConnString,dbUser,dbPwd);
}
} catch (Exception e) {
@ -116,6 +107,11 @@ public class RESOservlet extends HttpServlet
ArrayList<ResourceInfo> resources = new ArrayList<>();
// We are going to use a custom field definition to query Fields
FieldDefinition fieldDefinition = new FieldDefinition();
resources.add(fieldDefinition);
fieldDefinition.addResources(resources);
// Get all classes with constructors with 0 parameters. LookupDefinition should not work.
try
{
@ -133,10 +129,17 @@ public class RESOservlet extends HttpServlet
{
ctor.setAccessible(true);
ResourceInfo resource = (ResourceInfo)ctor.newInstance();
try
{
resource.findPrimaryKey(this.connect);
resources.add(resource);
resourceLookup.put(resource.getResourceName(), resource);
resource.findPrimaryKey(this.connect);
}
catch (Exception e)
{
LOG.error("Error with: "+resource.getResourceName()+" - "+e.getMessage());
}
}
}
}