diff --git a/.github/workflows/commit-dev.yml b/.github/workflows/commit-dev.yml index 3fec592..79ba694 100644 --- a/.github/workflows/commit-dev.yml +++ b/.github/workflows/commit-dev.yml @@ -15,6 +15,13 @@ jobs: env: ENVIRONMENT: dev DOCKER_BUILDKIT: 1 + COMPOSE_FILE: docker-compose.yml:./optional/docker-db-compose.yml + SQL_HOST: docker-mysql + SQL_USER: root + SQL_PASSWORD: root + SQL_DB_DRIVER: com.mysql.cj.jdbc.Driver + SQL_CONNECTION_STR: jdbc:mysql://docker-mysql/reso_data_dictionary_1_7?autoReconnect=true&maxReconnects=4 + CERT_REPORT_FILENAME: RESODataDictionary-1.7.metadata-report.json steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index ee006df..3e6a474 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ bin classes .DS_Store *.local +/*.sql +/*.json nb-configuration.xml .externalToolBuilders maven-eclipse.xml diff --git a/README.md b/README.md index d9289ab..22f7fd1 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Run the `run.sh` ## Access the Server -Assuming you're running the server locally, go to [http://localhost:8080/RESOservice-1.0/$metadata](http://localhost:8080/RESOservice-1.0/$metadata)\ +Assuming you're running the server locally, go to [http://localhost:8080/core/2.0.0/$metadata](http://localhost:8080/core/2.0.0/$metadata)\ Otherwise, you will have to replace `localhost` with the IP of your Docker machine. ## Running with a different database @@ -48,7 +48,7 @@ The `docker/docker-builder` file has a line commented out for Windows users, and This has not been tested. Anyone wanting to give feedback would be appreciated. -## BUILD FAILURES +## Build Failures In the case this happens, and you have fixed the source of the error and need to rebuild everything using the build scripts, you should delete any prior Docker containers. diff --git a/docker/scripts/build.sh b/docker/scripts/build.sh index 9ed5cc6..d543334 100644 --- a/docker/scripts/build.sh +++ b/docker/scripts/build.sh @@ -39,7 +39,7 @@ fi if gradle build then - cp build/libs/RESOservice-1.0.war ./target/ + cp build/libs/RESOservice-1.0.war ./target/core.war cp RESODataDictionary-1.7.metadata-report.json ./target/ else @@ -54,6 +54,7 @@ else exit else mvn package + mv ./target/RESOservice-1.0.war ./target/core.war cp RESODataDictionary-1.7.metadata-report.json ./target/ fi fi \ No newline at end of file diff --git a/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java b/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java index 1a20adf..e1ace57 100644 --- a/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java +++ b/src/main/java/org/reso/service/data/GenericEntityCollectionProcessor.java @@ -307,6 +307,7 @@ public class GenericEntityCollectionProcessor implements EntityCollectionProcess } for (Entity product :productList) { + // The getValue should already be a String, so the toString should just pass it through, while making the following assignment simple. String key = product.getProperty(primaryFieldName).getValue().toString(); HashMap enumValues = entities.get(key); CommonDataProcessing.setEntityEnums(enumValues,product,enumFields); diff --git a/src/main/java/org/reso/service/data/GenericEntityProcessor.java b/src/main/java/org/reso/service/data/GenericEntityProcessor.java index a808d6c..12282d7 100644 --- a/src/main/java/org/reso/service/data/GenericEntityProcessor.java +++ b/src/main/java/org/reso/service/data/GenericEntityProcessor.java @@ -32,10 +32,7 @@ import org.slf4j.LoggerFactory; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.*; import static org.reso.service.servlet.RESOservlet.resourceLookup; @@ -102,12 +99,23 @@ public class GenericEntityProcessor implements EntityProcessor response.setHeader(HttpHeader.CONTENT_TYPE, responseFormat.toContentTypeString()); } - protected Entity getData(EdmEntitySet edmEntitySet, List keyPredicates, ResourceInfo resource) throws ODataApplicationException { + + /** + * Reads data from a resource and returns it as a HashMap + * @param keyPredicates + * @param resource + * @return + */ + private HashMap getDataToHash(List keyPredicates, ResourceInfo resource) + { + return CommonDataProcessing.translateEntityToMap(this.getData(null, keyPredicates, resource)); + } + + protected Entity getData(EdmEntitySet edmEntitySet, List keyPredicates, ResourceInfo resource) { ArrayList fields = resource.getFieldList(); Entity product = null; - Map properties = System.getenv(); List enumFields = CommonDataProcessing.gatherEnumFields(resource); try { @@ -119,18 +127,21 @@ public class GenericEntityProcessor implements EntityProcessor // Result set get the result of the SQL query String queryString = null; - for (final UriParameter key : keyPredicates) + if (null!=keyPredicates) { - // key - String keyName = key.getName(); // .toLowerCase(); - String keyValue = key.getText(); - if (sqlCriteria==null) + for (final UriParameter key : keyPredicates) { - sqlCriteria = keyName + " = " + keyValue; - } - else - { - sqlCriteria = sqlCriteria + " and " + keyName + " = " + keyValue; + // key + String keyName = key.getName(); // .toLowerCase(); + String keyValue = key.getText(); + if (sqlCriteria==null) + { + sqlCriteria = keyName + " = " + keyValue; + } + else + { + sqlCriteria = sqlCriteria + " and " + keyName + " = " + keyValue; + } } } @@ -213,22 +224,23 @@ public class GenericEntityProcessor implements EntityProcessor DeserializerResult result = deserializer.entity(requestInputStream, edmEntityType); Entity requestEntity = result.getEntity(); // 2.2 do the creation in backend, which returns the newly created entity - //Entity createdEntity = storage.createEntityData(edmEntitySet, requestEntity); HashMap mappedObj = CommonDataProcessing.translateEntityToMap(requestEntity); - String primaryFieldName = resource.getPrimaryKeyName(); + List enumFields = CommonDataProcessing.gatherEnumFields(resource); - ArrayList enumValues = new ArrayList<>(); + HashMap enumValues = new HashMap<>(); for (FieldInfo field: enumFields) { + // We remove all entities that are collections to save to the lookup_value table separately. @TODO: save these values if (field.isCollection()) { String fieldName = field.getFieldName(); Object value = mappedObj.remove(fieldName); - enumValues.add(value); + enumValues.put(fieldName, value); } } saveData(resource, mappedObj); + saveEnumData(resource, enumValues); // 3. serialize the response (we have to return the created entity) ContextURL contextUrl = ContextURL.with().entitySet(edmEntitySet).build(); @@ -242,7 +254,8 @@ public class GenericEntityProcessor implements EntityProcessor //4. configure the response object response.setContent(serializedResponse.getContent()); response.setStatusCode(HttpStatusCode.CREATED.getStatusCode()); - response.setHeader(HttpHeader.CONTENT_TYPE, responseFormat.toContentTypeString()); } + response.setHeader(HttpHeader.CONTENT_TYPE, responseFormat.toContentTypeString()); + } private void saveData(ResourceInfo resource, HashMap mappedObj) @@ -300,6 +313,75 @@ public class GenericEntityProcessor implements EntityProcessor // Result set get the result of the SQL query } + private void saveEnumData(ResourceInfo resource, HashMap enumValues) + { + for (String key: enumValues.keySet() ) + { + Object value = enumValues.get(key); + saveEnumData(resource, key, value); + } + } + + + /** + * Save the Enum values for the enumObject for the resource. + * lookup_value table: + * +--------------------------+------------+------+-----+---------------------+-------------------------------+ + * | Field | Type | Null | Key | Default | Extra | + * +--------------------------+------------+------+-----+---------------------+-------------------------------+ + * | LookupValueKey | text | YES | | NULL | | + * | LookupValueKeyNumeric | bigint(20) | YES | | NULL | | + * | ResourceName | text | YES | | NULL | | + * | ResourceRecordKey | text | YES | | NULL | | + * | ResourceRecordKeyNumeric | bigint(20) | YES | | NULL | | + * | LookupKey | text | YES | | NULL | | + * | modificationTimestamp | timestamp | NO | | current_timestamp() | on update current_timestamp() | + * | FieldName | text | NO | | NULL | | + * +--------------------------+------------+------+-----+---------------------+-------------------------------+ + * @param resource + * @param values + */ + private void saveEnumData(ResourceInfo resource, String lookupEnumField, Object values) + { + String queryString = "insert into lookup_value"; + + /** + String value = resultSet.getString("LookupKey"); + String fieldName = resultSet.getString("FieldName"); + String resourceRecordKey = resultSet.getString("ResourceRecordKey"); + */ + + try + { + Statement statement = connect.createStatement(); + List columnNames = Arrays.asList("FieldName","LookupKey"); + + ArrayList valueArray; + + if (values instanceof ArrayList) + { + valueArray = (ArrayList) values; + } + else + { + ArrayList temp = new ArrayList(); + temp.add(values); + valueArray = temp; + } + + for (Object value : valueArray) + { + ArrayList columnValues = new ArrayList(Arrays.asList(lookupEnumField,value.toString())); + } + + } + catch (SQLException e) + { + LOG.error(e.getMessage()); + } + + } + @Override public void updateEntity(ODataRequest request, ODataResponse response, UriInfo uriInfo, ContentType requestFormat, ContentType responseFormat) throws ODataApplicationException, ODataLibraryException diff --git a/src/main/java/org/reso/service/data/common/CommonDataProcessing.java b/src/main/java/org/reso/service/data/common/CommonDataProcessing.java index 4227182..a51e77e 100644 --- a/src/main/java/org/reso/service/data/common/CommonDataProcessing.java +++ b/src/main/java/org/reso/service/data/common/CommonDataProcessing.java @@ -35,6 +35,13 @@ public class CommonDataProcessing private static final Logger LOG = LoggerFactory.getLogger(CommonDataProcessing.class); private static HashMap> resourceEnumFields = new HashMap<>(); + + /** + * This function will return the Enum fields for a given resource. + * It returns from the cache if found, otherwise it finds the Enum fields from the Field list and caches it for later use. + * @param resource + * @return List The Enum fields' FieldInfo values + */ public static List gatherEnumFields(ResourceInfo resource) { String resourceName = resource.getResourceName(); @@ -56,23 +63,35 @@ public class CommonDataProcessing } } + // Put it in the cache CommonDataProcessing.resourceEnumFields.put(resourceName, enumFields); return enumFields; } + + /** + * This will return the value for the field from the result set from the data source. + * @param field The field metadata + * @param resultSet The data source row + * @return A Java Object representing the value. It can be anything, but should be a simple representation for ease of manipulating. + * @throws SQLException in case of SQL error from the data source + */ public static Object getFieldValueFromRow(FieldInfo field, ResultSet resultSet) throws SQLException { String fieldName = field.getFieldName(); Object value = null; + // In case of a String if (field.getType().equals(EdmPrimitiveTypeKind.String.getFullQualifiedName())) { value = resultSet.getString(fieldName); } + // In case of a DateTime entry else if (field.getType().equals(EdmPrimitiveTypeKind.DateTimeOffset.getFullQualifiedName())) { value = resultSet.getTimestamp(fieldName); } + // @TODO: More will have to be added here, ie: Integers, as data comes in, we can extend this easily here. else { LOG.info("Field Name: "+field.getFieldName()+" Field type: "+field.getType()); @@ -81,16 +100,32 @@ public class CommonDataProcessing return value; } + + /** + * Builds an Entity from the row from the Resource's data source + * @param resultSet Data source row result + * @param resource The resource we're making an Entity for + * @param selectLookup An optional lookup of boolean flags that will only fill in the Entity values for entries with True lookup values + * @return An Entity representing the data source row + * @throws SQLException in case of SQL error from the data source + */ public static Entity getEntityFromRow(ResultSet resultSet, ResourceInfo resource, HashMap selectLookup) throws SQLException { String primaryFieldName = resource.getPrimaryKeyName(); ArrayList fields = resource.getFieldList(); + + // Lookup Key for the primary key String lookupKey = null; + // We only need to set the entity ID later if we're providing selectLookup and the primary field name is being requested + // @TODO: May need different logic here, ie: selectLookup==null || ... if (selectLookup!=null && selectLookup.get(primaryFieldName)!=null) { lookupKey = resultSet.getString(primaryFieldName); } + + // New entity to be populated Entity ent = new Entity(); + for (FieldInfo field : fields) { String fieldName = field.getODATAFieldName(); @@ -98,14 +133,17 @@ public class CommonDataProcessing if ( (selectLookup==null || selectLookup.containsKey(fieldName) )) { value = CommonDataProcessing.getFieldValueFromRow(field, resultSet); + // We only load Enums from the lookup_value table. @TODO: This may need revision to accommodate lookups on resource tables if (field instanceof EnumFieldInfo) { LOG.error("ENUMS currently only handles by values in lookup_value table. One must Define if this uses a key a numeric value."); } + // This is Enums that are bit masks, stored on the resource. else if (field.isCollection()) { ent.addProperty(new Property(null, fieldName, ValueType.ENUM, value)); } + // Simply put in primitive values as entity properties. else { ent.addProperty(new Property(null, fieldName, ValueType.PRIMITIVE, value)); @@ -113,6 +151,7 @@ public class CommonDataProcessing } } + // Set the entity ID if the lookupKey is provided in the select lookups if (lookupKey!=null) { ent.setId(createId(resource.getResourcesName(), lookupKey)); @@ -121,6 +160,16 @@ public class CommonDataProcessing return ent; } + + /** + * Returns a HashMap representation of a row from the data source, similar to the above function. + * Useful for building a simple Lookup cache, apart from Entities + * @param resultSet Data source row result + * @param resource The resource we're making an Entity for + * @param selectLookup An optional lookup of boolean flags that will only fill in the Entity values for entries with True lookup values + * @return A HashMap representing the data source row + * @throws SQLException in case of SQL error from the data source + */ public static HashMap getObjectFromRow(ResultSet resultSet, ResourceInfo resource, HashMap selectLookup) throws SQLException { String primaryFieldName = resource.getPrimaryKeyName(); @@ -150,6 +199,14 @@ public class CommonDataProcessing return ent; } + + /** + * For populating entity values Enums based on a potential non-sequential data source query results + * @param resultSet Data source row result + * @param entities A lookup of HashMap entities to be populated with Enum values + * @param enumFields The Enum fields to populate for the resource + * @throws SQLException in case of SQL error from the data source + */ public static void getEntityValues(ResultSet resultSet,HashMap> entities, List enumFields) throws SQLException { HashMap enumFieldLookup = new HashMap<>(); @@ -207,20 +264,26 @@ public class CommonDataProcessing } } - public static void setEntityEnums(HashMap enumValues, Entity entity, List enumFields) throws SQLException - { - HashMap enumFieldLookup = new HashMap<>(); + /** + * Translate the Enum values from a HashMap representation to an Entity representation + * @param enumValues The HashMap representation of the Enum values from the data source + * @param entity The Entity to populate with Enum values + * @param enumFields The Enum fields on the Entity we want values for + */ + public static void setEntityEnums(HashMap enumValues, Entity entity, List enumFields) + { for (FieldInfo field: enumFields) { EnumFieldInfo enumField = (EnumFieldInfo) field; String fieldName = enumField.getFieldName(); - long totalFlagValues = 3; + long totalFlagValues = 0; if (field.isFlags()) { try { + // Builds a bit flag representation of the multiple values. Object flagValues = enumValues.get(fieldName); ArrayList flagsArray = (ArrayList) flagValues; for (Object flagObj : flagsArray) @@ -229,18 +292,21 @@ public class CommonDataProcessing totalFlagValues = totalFlagValues + flagLong; } } - catch (Exception e) + catch (Exception e) // In case of casting error. "Should not happen" { LOG.error(e.getMessage()); } } + // There's many ways to represent Enums if (field.isCollection()) { + // As a Collection with bit flags if (field.isFlags()) { entity.addProperty(new Property(null, fieldName, ValueType.ENUM, totalFlagValues)); // @ToDo: This might not be compatible with anything... } + // A collection of Primitive types else { entity.addProperty(new Property(null, fieldName, ValueType.COLLECTION_PRIMITIVE, enumValues.get(fieldName))); @@ -248,10 +314,12 @@ public class CommonDataProcessing } else { + // Single value, bit flag representation if (field.isFlags()) { entity.addProperty(new Property(null, fieldName, ValueType.PRIMITIVE, totalFlagValues)); } + // Single value Primitive else { entity.addProperty(new Property(null, fieldName, ValueType.PRIMITIVE, enumValues.get(fieldName))); @@ -260,6 +328,12 @@ public class CommonDataProcessing } } + + /** + * Translates an Entity to a HashMap representation + * @param entity The Entity to turn into a HashMap + * @return The HashMap representation of the Entity + */ public static HashMap translateEntityToMap(Entity entity) { HashMap result = new HashMap<>(); @@ -276,6 +350,13 @@ public class CommonDataProcessing return result; } + + /** + * Loads all Resource entries into a List of HashMap representations of the entries. Useful for caching. + * @param connect The data source connection + * @param resource The Resource to load + * @return A List of HashMap representations of the entries + */ public static ArrayList> loadAllResource(Connection connect, ResourceInfo resource) { ArrayList fields = resource.getFieldList(); @@ -312,8 +393,15 @@ public class CommonDataProcessing } return productList; - } + + + /** + * Creates an unique URI identifier for the entity / id + * @param entitySetName Name of the Entity set + * @param id unique ID of the object + * @return unique URI identifier for the entity / id + */ private static URI createId(String entitySetName, Object id) { try { return new URI(entitySetName + "('" + id + "')"); diff --git a/src/main/java/org/reso/service/data/definition/LookupDefinition.java b/src/main/java/org/reso/service/data/definition/LookupDefinition.java index e14d87a..8e07960 100644 --- a/src/main/java/org/reso/service/data/definition/LookupDefinition.java +++ b/src/main/java/org/reso/service/data/definition/LookupDefinition.java @@ -61,10 +61,10 @@ public class LookupDefinition extends ResourceInfo fieldInfo = new FieldInfo("ModificationTimestamp", EdmPrimitiveTypeKind.DateTimeOffset.getFullQualifiedName()); list.add(fieldInfo); + /** //// Enum Test code EnumFieldInfo enumFieldInfo = new EnumFieldInfo("EnumTest", EdmPrimitiveTypeKind.Int64.getFullQualifiedName()); - /** enumFieldInfo.setLookupName("EnumTest"); //enumFieldInfo.setCollection(); enumFieldInfo.setFlags(); diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 4c1397b..5ceef3d 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -23,6 +23,6 @@ RESOservlet - /* + /2.0.0/* \ No newline at end of file