Make Report Generator Friendlier with Jenkins and Build Processes (#72)

* Added file generation to DD Metadata tests
* Updated most libraries and build.gradle accordingly
* Fixed annotation processor so it read Term instead of Type

build.gradle

@@ -10,7 +10,7 @@ plugins {
 }
 
 // Define the main class for the application
-mainClassName = 'org.reso.commander.Main'
+mainClassName = 'org.reso.commander.App'
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 compileJava.options.encoding = 'UTF-8'
@@ -22,8 +22,8 @@ repositories {
 }
 
 dependencies {
-  compile 'com.google.guava:guava:28.2-jre'
-  compile 'com.google.inject:guice:4.2.3'
+  compile 'com.google.guava:guava:30.1.1-jre'
+  compile 'com.google.inject:guice:5.0.1'
   compile 'commons-cli:commons-cli:1.4'
 
   compile 'org.apache.logging.log4j:log4j-api:2.13.0'
@@ -35,22 +35,23 @@ dependencies {
   compile 'org.apache.olingo:odata-commons-core:4.8.0'
   compile 'org.apache.olingo:odata-client-core:4.8.0'
 
-  compile 'org.apache.poi:poi:3.17'
-  compile 'org.apache.poi:poi-ooxml:3.17'
+  compile 'org.apache.poi:poi:5.0.0'
+  compile 'org.apache.poi:poi-ooxml:5.0.0'
 
-  compile 'io.rest-assured:rest-assured:4.2.0'
-  compile 'io.rest-assured:json-path:4.2.0'
-  compile 'io.rest-assured:json-schema-validator:4.2.0'
+  compile 'io.rest-assured:rest-assured:4.3.3'
+  compile 'io.rest-assured:json-path:4.3.3'
+  compile 'io.rest-assured:json-schema-validator:4.3.3'
 
-  compile 'io.cucumber:cucumber-java8:5.4.0'
-  compile 'io.cucumber:cucumber-java:5.4.0'
-  compile 'io.cucumber:cucumber-junit:5.4.0'
-  compile 'io.cucumber:cucumber-guice:5.4.0'
-  compile 'io.cucumber:cucumber-core:5.4.0'
+  compile 'io.cucumber:cucumber-java8:6.10.2'
+  compile 'io.cucumber:cucumber-java:6.10.2'
+  compile 'io.cucumber:cucumber-junit:6.10.2'
+  compile 'io.cucumber:cucumber-guice:6.10.2'
+  compile 'io.cucumber:cucumber-core:6.10.2'
 
-  compile 'net.masterthought:cucumber-reporting:5.1.1'
+  compile 'net.masterthought:cucumber-reporting:5.5.2'
 
-  compile 'com.networknt:json-schema-validator:1.0.35'
+  //TODO: choose one schema validator between this and rest-assured
+  compile 'com.networknt:json-schema-validator:1.0.51'
   compile 'com.google.code.gson:gson:2.8.6'
   compile 'org.apache.commons:commons-text:1.9'
 
@@ -64,18 +65,19 @@ configurations {
 
 jar {
   manifest {
-    attributes(
-        'Class-Path': configurations.compile.collect { it.getName() }.join(' '),
-        'Main-Class': 'org.reso.commander.App'
-    )
+    attributes "Main-Class": mainClassName
   }
 
-  // generate "fat jar" by default
   from {
     configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
   }
+  exclude 'META-INF/*.RSA'
+  exclude 'META-INF/*.SF'
+  exclude 'META-INF/*.DSA'
 }
 
+
+
 // don't suppress warnings or deprecation notices
 tasks.withType(JavaCompile) {
   options.compilerArgs << '-Xlint:unchecked'
@@ -330,3 +332,4 @@ task _jenkins_testWebApiServer_1_0_2_Core() {
     }
   }
 }
+

src/main/java/org/reso/certification/containers/WebAPITestContainer.java

@@ -20,7 +20,6 @@ import org.apache.olingo.client.api.domain.ClientEntitySet;
 import org.apache.olingo.client.api.edm.xml.XMLMetadata;
 import org.apache.olingo.client.api.uri.QueryOption;
 import org.apache.olingo.commons.api.edm.Edm;
-import org.apache.olingo.commons.api.edm.provider.CsdlEnumType;
 import org.apache.olingo.commons.api.edm.provider.CsdlProperty;
 import org.apache.olingo.commons.api.format.ContentType;
 import org.reso.commander.Commander;
@@ -100,7 +99,6 @@ public final class WebAPITestContainer implements TestContainer {
 
   //singleton variables
   private static final AtomicReference<Map<String, Map<String, CsdlProperty>>> fieldMap = new AtomicReference<>();
-  private static final AtomicReference<Map<String, Map<String, CsdlEnumType>>> enumMap = new AtomicReference<>();
 
   /**
    * Initializes the container with a singleton instance of the RESO Commander

src/main/java/org/reso/certification/reporting/CertificationReportGenerator.java

@@ -28,7 +28,7 @@ public class CertificationReportGenerator {
   private static final boolean USE_MINIMAL_REPORT = Boolean.parseBoolean(System.getProperty("minimal", "false"));
   private static final String DEFAULT_REPORT_DESCRIPTION = "Certification Report";
   private static final String projectName = System.getProperty("reportDescription", DEFAULT_REPORT_DESCRIPTION);
-  private static final String MINIMAL_JSON_EXTENSION = ".minimal.json";
+  private static final String MINIMAL_JSON_EXTENSION = "minimal.json";
 
   public static void main(String[] args) {
 

src/main/java/org/reso/certification/stepdefs/DataDictionary.java

@@ -184,6 +184,9 @@ public class DataDictionary {
         container.validateMetadata();
         areMetadataValid = container.hasValidMetadata();
 
+        //create metadata report
+        Commander.generateMetadataReport(container.getEdm());
+
       } catch (IOException e) {
         failAndExitWithErrorMessage(getDefaultErrorMessage(e), scenario);
       }
@@ -215,6 +218,9 @@ public class DataDictionary {
 
       //if we have gotten to this point without exceptions, then metadata are valid
       areMetadataValid = container.hasValidMetadata();
+
+      //create metadata report
+      Commander.generateMetadataReport(container.getEdm());
     }
   }
 
@@ -260,7 +266,7 @@ public class DataDictionary {
     foundStandardMembers.set(getFoundStandardMembers(foundMembers.get(), dataTable));
 
     if (foundStandardMembers.get().size() == 0) {
-      scenario.write("No RESO Standard Enumerations found for field: " + fieldName);
+      scenario.log("No RESO Standard Enumerations found for field: " + fieldName);
     }
   }
 
@@ -461,10 +467,10 @@ public class DataDictionary {
         ? container.getFieldMap(currentResourceName.get()).get(fieldName).getPrecision() : null;
 
     if (!Objects.equals(precision, suggestedPrecision)) {
-      scenario.write("Precision for field " + fieldName +  " SHOULD be equal to the RESO Suggested Max Precision of " + suggestedPrecision
+      scenario.log("Precision for field " + fieldName +  " SHOULD be equal to the RESO Suggested Max Precision of " + suggestedPrecision
           + " but was " + precision);
     } else {
-      scenario.write("Precision for field " + fieldName +  " is equal to the RESO Suggested Max Scale of " + suggestedPrecision);
+      scenario.log("Precision for field " + fieldName +  " is equal to the RESO Suggested Max Scale of " + suggestedPrecision);
     }
   }
 
@@ -475,10 +481,10 @@ public class DataDictionary {
         ? container.getFieldMap(currentResourceName.get()).get(fieldName).getScale() : null;
 
     if (!Objects.equals(scale, suggestedMaxScale)) {
-      scenario.write("Scale for field " + fieldName +  " SHOULD be equal to the RESO Suggested Max Scale of " + suggestedMaxScale
+      scenario.log("Scale for field " + fieldName +  " SHOULD be equal to the RESO Suggested Max Scale of " + suggestedMaxScale
           + " but was " + scale);
     } else {
-      scenario.write("Scale for field " + fieldName +  " is equal to the RESO Suggested Max Scale of " + suggestedMaxScale);
+      scenario.log("Scale for field " + fieldName +  " is equal to the RESO Suggested Max Scale of " + suggestedMaxScale);
     }
   }
 
@@ -489,10 +495,10 @@ public class DataDictionary {
         ? container.getFieldMap(currentResourceName.get()).get(fieldName).getMaxLength() : null;
 
     if (!Objects.equals(length, suggestedMaxLength)) {
-      scenario.write("Length for field " + fieldName + " SHOULD be equal to the RESO Suggested Max Length of " + suggestedMaxLength
+      scenario.log("Length for field " + fieldName + " SHOULD be equal to the RESO Suggested Max Length of " + suggestedMaxLength
           + " but was " + length);
     } else {
-      scenario.write("Length for field " + fieldName +  " is equal to the RESO Suggested Max Length of " + length);
+      scenario.log("Length for field " + fieldName +  " is equal to the RESO Suggested Max Length of " + length);
     }
   }
 

src/main/java/org/reso/commander/Commander.java

@@ -184,19 +184,29 @@ public class Commander {
    */
   public static String generateMetadataReport(Edm metadata, String fileName) {
     final String DEFAULT_FILENAME = "metadata-report.json";
-    MetadataReport report = new MetadataReport(metadata);
-    GsonBuilder gsonBuilder = new GsonBuilder().setPrettyPrinting();
-    gsonBuilder.registerTypeAdapter(MetadataReport.class, report);
 
     try {
-      FileUtils.copyInputStreamToFile(new ByteArrayInputStream(gsonBuilder.create().toJson(report).getBytes()),
-          new File(fileName != null ? fileName.replaceAll(".edmx|.xml", EMPTY_STRING)  + ".metadata-report.json"
-              : DEFAULT_FILENAME));
+      MetadataReport report = new MetadataReport(metadata);
+      GsonBuilder gsonBuilder = new GsonBuilder().setPrettyPrinting();
+      gsonBuilder.registerTypeAdapter(MetadataReport.class, report);
+
+      File targetReportFile;
+
+      if (fileName != null) {
+        targetReportFile = new File(fileName.replaceAll(".edmx|.xml", EMPTY_STRING)  + ".metadata-report.json");
+      } else {
+        //place unnamed files in the build directory
+        targetReportFile = new File("build", DEFAULT_FILENAME);
+      }
+
+      FileUtils.copyInputStreamToFile(new ByteArrayInputStream(gsonBuilder.create().toJson(report).getBytes()), targetReportFile);
+
+      return report.toString();
     } catch (Exception ex) {
       LOG.error(getDefaultErrorMessage(ex));
     }
 
-    return report.toString();
+    return null;
   }
 
   public static String generateMetadataReport(Edm metadata) {

src/main/java/org/reso/commander/common/TestUtils.java

@@ -961,7 +961,7 @@ public final class TestUtils {
 
   public static void failAndExitWithErrorMessage(String msg, Scenario scenario) {
     if (scenario != null) {
-      scenario.write(getDefaultErrorMessage(msg));
+      scenario.log(getDefaultErrorMessage(msg));
     }
     System.exit(NOT_OK);
   }

src/main/java/org/reso/models/MetadataReport.java

@@ -1,16 +1,24 @@
 package org.reso.models;
 
 import com.google.gson.*;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.olingo.client.core.edm.xml.ClientCsdlAnnotation;
 import org.apache.olingo.commons.api.edm.*;
+import org.apache.olingo.commons.core.edm.EdmAnnotationImpl;
 import org.reso.commander.common.Utils;
 
+import java.lang.reflect.Field;
 import java.lang.reflect.Type;
 import java.util.Date;
 import java.util.List;
 
 import static org.reso.commander.Commander.REPORT_DIVIDER;
+import static org.reso.commander.common.ErrorMsg.getDefaultErrorMessage;
 
 public class MetadataReport implements JsonSerializer<MetadataReport> {
+  private static final Logger LOG = LogManager.getLogger(MetadataReport.class);
+
   private Edm metadata;
 
   private MetadataReport() {
@@ -59,6 +67,7 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
         DEFAULT_VALUE_KEY = "defaultValue",
         UNICODE_KEY = "unicode",
         TYPE_KEY = "type",
+        TERM_KEY = "term",
         VALUE_KEY= "value",
         ANNOTATIONS_KEY = "annotations",
         FIELDS_KEY = "fields";
@@ -87,9 +96,9 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
             reportBuilder.append("\n");
             reportBuilder.append("Annotations:");
             annotations.forEach(annotation -> {
-              if (annotation.getAsJsonObject().get(TYPE_KEY) != null) {
-                reportBuilder.append("\n\tType: ");
-                reportBuilder.append(annotation.getAsJsonObject().get(TYPE_KEY));
+              if (annotation.getAsJsonObject().get(TERM_KEY) != null) {
+                reportBuilder.append("\n\tTerm: ");
+                reportBuilder.append(annotation.getAsJsonObject().get(TERM_KEY));
               }
 
               if (annotation.getAsJsonObject().get(VALUE_KEY) != null) {
@@ -108,9 +117,19 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
     public JsonElement serialize(FieldJson src, Type typeOfSrc, JsonSerializationContext context) {
       JsonObject field = new JsonObject();
 
+
       field.addProperty(RESOURCE_NAME_KEY, src.resourceName);
       field.addProperty(FIELD_NAME_KEY, src.edmElement.getName());
-      field.addProperty(TYPE_KEY, src.edmElement.getType().getFullQualifiedName().getFullQualifiedNameAsString());
+
+      String typeName = null;
+      try {
+        typeName = src.edmElement.getType().getFullQualifiedName().getFullQualifiedNameAsString();
+        field.addProperty(TYPE_KEY, typeName);
+      } catch (Exception ex) {
+        LOG.error(getDefaultErrorMessage("Field Name:", src.edmElement.getName(), ex.toString()));
+        field.addProperty(TYPE_KEY, "UNDEFINED");
+      }
+
       field.addProperty(NULLABLE_KEY, ((EdmProperty) src.edmElement).isNullable());
       field.addProperty(MAX_LENGTH_KEY, ((EdmProperty) src.edmElement).getMaxLength());
       field.addProperty(SCALE_KEY, ((EdmProperty) src.edmElement).getScale());
@@ -128,7 +147,10 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
         annotations.forEach(edmAnnotation -> {
           JsonObject annotation = new JsonObject();
           if (edmAnnotation.getTerm() != null) {
-            annotation.addProperty(TYPE_KEY, edmAnnotation.getTerm().getFullQualifiedName().getFullQualifiedNameAsString());
+            annotation.addProperty(TERM_KEY, edmAnnotation.getTerm().getFullQualifiedName().getFullQualifiedNameAsString());
+          } else {
+            SneakyAnnotationReader sneakyAnnotationReader = new SneakyAnnotationReader(edmAnnotation);
+            annotation.addProperty(TERM_KEY, sneakyAnnotationReader.getTerm());
           }
 
           if (edmAnnotation.getExpression() != null) {
@@ -142,6 +164,36 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
     }
   }
 
+  static class SneakyAnnotationReader {
+    Class object;
+    Field field;
+    EdmAnnotationImpl edmAnnotationImpl;
+    ClientCsdlAnnotation clientCsdlAnnotation;
+
+    public SneakyAnnotationReader(EdmAnnotation edmAnnotation) {
+      try {
+        edmAnnotationImpl = ((EdmAnnotationImpl)edmAnnotation);
+
+        // create an object of the class named Class
+        object = edmAnnotationImpl.getClass();
+
+        // access the private variable
+        field = object.getDeclaredField("annotation");
+        // make private field accessible
+        field.setAccessible(true);
+
+        clientCsdlAnnotation = (ClientCsdlAnnotation) field.get(edmAnnotationImpl);
+
+      } catch (Exception ex) {
+        LOG.error(ex);
+      }
+    }
+
+    public String getTerm() {
+      return clientCsdlAnnotation.getTerm();
+    }
+  }
+
   /**
    * LookupJson uses a JSON payload with the following structure:
    *
@@ -181,9 +233,9 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
             reportBuilder.append("\n");
             reportBuilder.append("Annotations:");
             annotations.forEach(annotation -> {
-              if (annotation.getAsJsonObject().get(TYPE_KEY) != null) {
-                reportBuilder.append("\n\tType: ");
-                reportBuilder.append(annotation.getAsJsonObject().get(TYPE_KEY));
+              if (annotation.getAsJsonObject().get(FieldJson.TERM_KEY) != null) {
+                reportBuilder.append("\n\tTerm: ");
+                reportBuilder.append(annotation.getAsJsonObject().get(FieldJson.TERM_KEY));
               }
 
               if (annotation.getAsJsonObject().get(VALUE_KEY) != null) {
@@ -210,7 +262,10 @@ public class MetadataReport implements JsonSerializer<MetadataReport> {
         src.edmEnumType.getMember(memberName).getAnnotations().forEach(edmAnnotation -> {
           JsonObject annotation = new JsonObject();
           if (edmAnnotation.getTerm() != null) {
-            annotation.addProperty(TYPE_KEY, edmAnnotation.getTerm().getFullQualifiedName().getFullQualifiedNameAsString());
+            annotation.addProperty(FieldJson.TERM_KEY, edmAnnotation.getTerm().getFullQualifiedName().getFullQualifiedNameAsString());
+          } else {
+            SneakyAnnotationReader sneakyAnnotationReader = new SneakyAnnotationReader(edmAnnotation);
+            annotation.addProperty(FieldJson.TERM_KEY, sneakyAnnotationReader.getTerm());
           }
 
           if (edmAnnotation.getExpression() != null) {