#100: Refactor to support flattened sheets

This commit is contained in:
darnjo 2023-03-13 17:25:02 -07:00
parent 6228cc4251
commit 5d8c746a1d
11 changed files with 467 additions and 516 deletions

2
.gitignore vendored
View File

@ -7,3 +7,5 @@ build/
*.log
*.iml
.run/
.DS_Store

View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="generateDDAcceptanceTests" type="Application" factoryName="Application">
<option name="ALTERNATIVE_JRE_PATH" value="1.8" />
<option name="ALTERNATIVE_JRE_PATH" value="/Library/Java/JavaVirtualMachines/openjdk-11.jdk/Contents/Home" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
<option name="MAIN_CLASS_NAME" value="org.reso.commander.App" />
<module name="web-api-commander.main" />

View File

@ -21,10 +21,10 @@ dependencies {
implementation 'com.google.inject:guice:5.1.0'
implementation 'commons-cli:commons-cli:1.5.0'
implementation 'org.apache.logging.log4j:log4j-api:2.18.0'
implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.18.0'
implementation 'org.apache.logging.log4j:log4j-1.2-api:2.18.0'
implementation 'org.apache.logging.log4j:log4j-core:2.18.0'
implementation 'org.apache.logging.log4j:log4j-api:2.20.0'
implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.20.0'
implementation 'org.apache.logging.log4j:log4j-1.2-api:2.20.0'
implementation 'org.apache.logging.log4j:log4j-core:2.20.0'
implementation 'org.apache.olingo:odata-client-api:4.9.0'
implementation 'org.apache.olingo:odata-commons-core:4.9.0'
@ -57,15 +57,20 @@ configurations {
}
jar {
duplicatesStrategy = DuplicatesStrategy.INCLUDE
manifest {
attributes "Main-Class": mainClassName
attributes "Multi-Release": true
}
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
from {
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
}
exclude 'META-INF/*.RSA'
exclude 'META-INF/*.SF'
exclude 'META-INF/*.DSA'
}

View File

@ -4,7 +4,7 @@
* The settings file is used to specify which projects to include in your build.
*
* Detailed information about configuring a multi-project build in Gradle can be found
* in the user manual at https://docs.gradle.org/5.2.1/userguide/multi_project_builds.html
* in the user manual at https://docs.gradle.org/8.0.2/userguide/multi_project_builds.html
* This project uses @Incubating APIs which are subject to change.
*/
rootProject.setName('web-api-commander')
rootProject.name = 'web-api-commander'

View File

@ -112,8 +112,8 @@ public class BDDProcessor extends WorksheetProcessor {
//use this to add each field name tag
//tags.add(field.getStandardName());
if (field.getParentResourceName() != null && field.getParentResourceName().length() > 0) {
tags.add(field.getParentResourceName());
if (field.getResourceName() != null && field.getResourceName().length() > 0) {
tags.add(field.getResourceName());
}
tags.addAll(field.getPropertyTypes());
@ -133,7 +133,7 @@ public class BDDProcessor extends WorksheetProcessor {
if (field.getSynonyms().size() > 0) {
template += " Given that the following synonyms for \"" + field.getStandardName()
+ "\" DO NOT exist in the \"" + field.getParentResourceName() + "\" metadata\n" +
+ "\" DO NOT exist in the \"" + field.getResourceName() + "\" metadata\n" +
field.getSynonyms().stream()
.map(synonym -> padLeft("| " + synonym + " |\n", EXAMPLES_PADDING_AMOUNT)).collect(Collectors.joining());
}
@ -146,7 +146,7 @@ public class BDDProcessor extends WorksheetProcessor {
return "\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Boolean\" data type\n";
}
@ -156,7 +156,7 @@ public class BDDProcessor extends WorksheetProcessor {
return "\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Date\" data type\n";
}
@ -179,7 +179,7 @@ public class BDDProcessor extends WorksheetProcessor {
"\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Decimal\" data type\n";
//TODO Length is actually scale for Decimal fields by the DD! :/
@ -203,7 +203,7 @@ public class BDDProcessor extends WorksheetProcessor {
return "\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Integer\" data type\n";
}
@ -231,7 +231,7 @@ public class BDDProcessor extends WorksheetProcessor {
"\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Multiple Enumeration\" data type\n";
}
@ -242,7 +242,7 @@ public class BDDProcessor extends WorksheetProcessor {
"\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Single Enumeration\" data type\n";
}
@ -252,7 +252,7 @@ public class BDDProcessor extends WorksheetProcessor {
"\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"String\" data type\n";
if (field.getSuggestedMaxLength() != null)
@ -268,7 +268,7 @@ public class BDDProcessor extends WorksheetProcessor {
return "\n " + buildTags(field).stream().map(tag -> "@" + tag).collect(Collectors.joining(SINGLE_SPACE)) + "\n" +
" Scenario: " + field.getStandardName() + "\n" +
generateSynonymsMarkup(field) +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getParentResourceName() + "\" metadata\n" +
" When \"" + field.getStandardName() + "\" exists in the \"" + field.getResourceName() + "\" metadata\n" +
" Then \"" + field.getStandardName() + "\" MUST be \"Timestamp\" data type\n";
}
}

View File

@ -10,8 +10,8 @@ public class DDCacheProcessor extends WorksheetProcessor {
new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>()));
private void addToFieldCache(ReferenceStandardField field) {
standardFieldCache.get().putIfAbsent(field.getParentResourceName(), new LinkedHashMap<>());
standardFieldCache.get().get(field.getParentResourceName()).put(field.getStandardName(), field);
standardFieldCache.get().putIfAbsent(field.getResourceName(), new LinkedHashMap<>());
standardFieldCache.get().get(field.getResourceName()).put(field.getStandardName(), field);
}
public Map<String, Map<String, ReferenceStandardField>> getStandardFieldCache() {

View File

@ -31,41 +31,28 @@ public class DataDictionaryCodeGenerator {
/**
* Generates Data Dictionary references for local workbook instance using the configured WorksheetProcessor
*
* TODO: convert to .parallelStream()
*/
public void processWorksheets() {
Sheet currentWorksheet, standardResourcesWorksheet;
int sheetIndex, rowIndex;
final int ROW_HEADER_INDEX = 0, FIRST_ROW_INDEX = 1;
final String STANDARD_RELATIONSHIPS_WORKSHEET = "Standard Relationships";
Sheet fieldsWorksheet;
final int FIRST_ROW_INDEX = 1;
final String FIELDS_WORKSHEET = "Fields";
try {
standardResourcesWorksheet = workbook.getSheet(STANDARD_RELATIONSHIPS_WORKSHEET);
assert standardResourcesWorksheet != null;
fieldsWorksheet = workbook.getSheet(FIELDS_WORKSHEET);
assert fieldsWorksheet != null;
processor.beforeResourceSheetProcessed(fieldsWorksheet);
processor.buildStandardRelationships(standardResourcesWorksheet);
processor.wellKnownStandardFieldHeaderMap = buildWellKnownStandardFieldHeaderMap(fieldsWorksheet);
processor.processResourceSheet(fieldsWorksheet);
//workbook consists of many sheets, process only the ones that have the name of a well-known resource
for (sheetIndex = ROW_HEADER_INDEX; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
currentWorksheet = workbook.getSheetAt(sheetIndex);
if (DataDictionaryMetadata.v1_7.WELL_KNOWN_RESOURCES.contains(currentWorksheet.getSheetName()) && currentWorksheet.getPhysicalNumberOfRows() > 1) {
processor.beforeResourceSheetProcessed(currentWorksheet);
processor.wellKnownStandardFieldHeaderMap = buildWellKnownStandardFieldHeaderMap(currentWorksheet);
processor.processResourceSheet(currentWorksheet);
//starts at row 1 to skip header row
for (rowIndex = FIRST_ROW_INDEX; rowIndex < currentWorksheet.getPhysicalNumberOfRows(); rowIndex++) {
if (currentWorksheet.getRow(rowIndex) != null) {
processor.processResourceRow(currentWorksheet.getRow(rowIndex));
}
}
processor.afterResourceSheetProcessed(currentWorksheet);
//starts at row 1 to skip header row
for (int rowIndex = FIRST_ROW_INDEX; rowIndex < fieldsWorksheet.getPhysicalNumberOfRows(); rowIndex++) {
if (fieldsWorksheet.getRow(rowIndex) != null) {
processor.processResourceRow(fieldsWorksheet.getRow(rowIndex));
}
}
processor.afterResourceSheetProcessed(fieldsWorksheet);
processor.generateOutput();
} catch (Exception ex) {
LOG.info(ex);

View File

@ -175,7 +175,7 @@ public class EDMXProcessor extends WorksheetProcessor {
StringBuilder content = new StringBuilder();
content.append("<Schema xmlns=\"http://docs.oasis-open.org/odata/ns/edm\" Namespace=\"" + RESO_NAMESPACE + "\">");
//iterate through each of the found resources and generate their edm:EntityType content content
//iterate through each of the found resources and generate their edm:EntityType content
resourceTemplates.forEach((resourceName, templateContent) -> {
content.append(templateContent);
});
@ -377,10 +377,10 @@ public class EDMXProcessor extends WorksheetProcessor {
}
public static String buildEnumTypeSingleMember(ReferenceStandardField field) {
if (field == null || field.getLookup() == null) return EMPTY_STRING;
if (!field.getLookup().toLowerCase().contains("lookups")) return EMPTY_STRING;
if (field == null || field.getLookupName() == null) return EMPTY_STRING;
if (!field.getLookupName().toLowerCase().contains("lookups")) return EMPTY_STRING;
String lookupName = field.getLookup().replace("Lookups", "").trim();
String lookupName = field.getLookupName().replace("Lookups", "").trim();
return ""
+ "<Property Name=\"" + field.getStandardName() + "\" Type=\"" + RESO_NAMESPACE + ".enums." + lookupName + "\" >"
+ buildDisplayNameAnnotation(field.getDisplayName())
@ -390,8 +390,8 @@ public class EDMXProcessor extends WorksheetProcessor {
}
public static String buildEnumTypeMultiMember(ReferenceStandardField field) {
if (field == null || field.getLookup() == null) return EMPTY_STRING;
if (!field.getLookup().toLowerCase().contains("lookups")) return EMPTY_STRING;
if (field == null || field.getLookupName() == null) return EMPTY_STRING;
if (!field.getLookupName().toLowerCase().contains("lookups")) return EMPTY_STRING;
return ""
+ "<Property Name=\"" + field.getStandardName()
+ "\" Type=\"Collection(" + RESO_NAMESPACE + ".enums." + field.getLookupName() + ")\">"

View File

@ -2,10 +2,7 @@ package org.reso.certification.codegen;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.reso.commander.common.Utils;
import org.reso.models.ReferenceStandardField;
@ -16,427 +13,390 @@ import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.junit.Assert.assertTrue;
import static java.lang.System.exit;
import static org.reso.certification.codegen.WorksheetProcessor.WELL_KNOWN_DATA_TYPES.*;
import static org.reso.certification.codegen.WorksheetProcessor.WELL_KNOWN_FIELD_HEADERS.COLLECTION;
import static org.reso.certification.codegen.WorksheetProcessor.WELL_KNOWN_FIELD_HEADERS.STANDARD_NAME;
import static org.reso.commander.common.DataDictionaryMetadata.v1_7.LOOKUP_FIELDS_AND_VALUES;
import static org.reso.commander.common.ErrorMsg.getDefaultErrorMessage;
public abstract class WorksheetProcessor {
//TODO: make this a dynamic property based on DD version
public static final String REFERENCE_WORKSHEET = "RESODataDictionary-1.7.xlsx";
private static final Logger LOG = LogManager.getLogger(WorksheetProcessor.class);
static final Map<String, String> resourceTemplates = new LinkedHashMap<>();
static final Map<String, Set<ReferenceStandardLookup>> standardEnumerationsMap = new LinkedHashMap<>();
static final Map<String, Map<String, ReferenceStandardField>> standardFieldsMap = new LinkedHashMap<>(new LinkedHashMap<>());
private static final Logger LOG = LogManager.getLogger(WorksheetProcessor.class);
String referenceDocument = null;
StringBuffer markup;
Sheet sheet;
String startTimestamp;
//TODO: make this a dynamic property based on DD version
public static final String REFERENCE_WORKSHEET = "RESODataDictionary-1.7-Flattened.xlsx";
Map<String, Integer> wellKnownStandardFieldHeaderMap = new LinkedHashMap<>();
Map<String, Integer> wellKnownStandardEnumerationHeaderMap = new LinkedHashMap<>();
static final Map<String, String> resourceTemplates = new LinkedHashMap<>();
static final Map<String, Set<ReferenceStandardLookup>> standardEnumerationsMap = new LinkedHashMap<>();
static final Map<String, Map<String, ReferenceStandardField>> standardFieldsMap = new LinkedHashMap<>(new LinkedHashMap<>());
private final List<ReferenceStandardRelationship> referenceStandardRelationships = new ArrayList<>();
String referenceDocument = null;
StringBuffer markup;
Sheet sheet;
String startTimestamp;
Map<String, Integer> wellKnownStandardFieldHeaderMap = new LinkedHashMap<>();
Map<String, Integer> wellKnownStandardEnumerationHeaderMap = new LinkedHashMap<>();
private final List<ReferenceStandardRelationship> referenceStandardRelationships = new ArrayList<>();
static final int
TARGET_RESOURCE = 0,
TARGET_RESOURCE_KEY = 1,
TARGET_STANDARD_NAME = 2,
CARDINALITY = 3,
SOURCE_RESOURCE = 4,
SOURCE_RESOURCE_KEY = 5;
public WorksheetProcessor() {
startTimestamp = Utils.getTimestamp();
markup = new StringBuffer();
}
public static Map<String, Integer> buildWellKnownStandardFieldHeaderMap(Sheet sheet) {
Map<String, Integer> headerMap = new LinkedHashMap<>();
sheet.getRow(0).cellIterator().forEachRemaining(cell ->
headerMap.put(cell.getStringCellValue(), cell.getColumnIndex()));
return headerMap;
}
public Integer getWellKnownStandardFieldIndex(String wellKnownStandardFieldKey) {
return wellKnownStandardFieldHeaderMap.get(wellKnownStandardFieldKey);
}
public Integer getWellKnownStandardEnumerationIndex(String wellKnownStandardEnumerationKey) {
return wellKnownStandardEnumerationHeaderMap.get(wellKnownStandardEnumerationKey);
}
public void buildWellKnownStandardEnumerationHeaderMap(Sheet sheet) {
wellKnownStandardEnumerationHeaderMap = new LinkedHashMap<>();
sheet.getRow(0).cellIterator().forEachRemaining(cell ->
wellKnownStandardEnumerationHeaderMap.put(cell.getStringCellValue(), cell.getColumnIndex()));
}
public List<ReferenceStandardRelationship> getStandardRelationships() {
return this.referenceStandardRelationships;
}
public static Integer getIntegerValue(Integer index, Row row, Integer defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
Integer value;
DataFormatter formatter = new DataFormatter();
try {
value = Integer.parseInt(formatter.formatCellValue(row.getCell(index)));
} catch (Exception ex) {
value = defaultValue;
public WorksheetProcessor() {
LOG.info("Using Data Dictionary Reference sheet: " + REFERENCE_WORKSHEET);
startTimestamp = Utils.getTimestamp();
markup = new StringBuffer();
}
return value;
}
public static Integer getIntegerValue(Integer index, Row row) {
return getIntegerValue(index, row, null);
}
public static String getStringValue(Integer index, Row row, String defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
String value;
DataFormatter formatter = new DataFormatter();
try {
value = formatter.formatCellValue(row.getCell(index));
} catch (Exception ex) {
value = defaultValue;
public static Map<String, Integer> buildWellKnownStandardFieldHeaderMap(Sheet sheet) {
Map<String, Integer> headerMap = new LinkedHashMap<>();
sheet.getRow(0).cellIterator().forEachRemaining(cell ->
headerMap.put(cell.getStringCellValue(), cell.getColumnIndex()));
return headerMap;
}
return value;
}
public static String getStringValue(Integer index, Row row) {
return getStringValue(index, row, null);
}
public static Boolean getBooleanValue(Integer index, Row row, Boolean defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
final String BOOLEAN_VALUE = "yes";
Boolean value = false;
String cellValue;
DataFormatter formatter = new DataFormatter();
try {
cellValue = formatter.formatCellValue(row.getCell(index));
if (cellValue.toLowerCase().contains(BOOLEAN_VALUE)) {
value = true;
}
} catch (Exception ex) {
value = defaultValue;
}
return value;
}
public static Boolean getBooleanValue(Integer index, Row row) {
return getBooleanValue(index, row, false);
}
public static List<String> getArrayValue(Integer index, Row row, List<String> defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
DataFormatter formatter = new DataFormatter();
String cellValue;
List<String> value = new ArrayList<>();
try {
cellValue = formatter.formatCellValue(row.getCell(index));
if (cellValue != null && cellValue.length() > 0) {
//LOG.info("Cell index is: " + index + ", cell value is: " + cellValue);
value = Arrays.stream(cellValue
.replace(" ", "").split(","))
.map(String::trim)
.filter(item -> item.length() > 0)
.collect(Collectors.toList());
}
} catch (Exception ex) {
value = defaultValue;
}
return value;
}
public static List<String> getArrayValue(Integer index, Row row) {
return getArrayValue(index, row, new ArrayList<>());
}
public ReferenceStandardRelationship deserializeStandardRelationshipRow(Row row) {
ReferenceStandardRelationship referenceStandardRelationship = null;
try {
referenceStandardRelationship = ReferenceStandardRelationship.Builder.build(
row.getCell(TARGET_RESOURCE).getStringCellValue(),
row.getCell(TARGET_RESOURCE_KEY, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).getStringCellValue(),
row.getCell(TARGET_STANDARD_NAME).getStringCellValue(),
ReferenceStandardRelationship.Cardinality.stream().filter(cardinality ->
cardinality.getRelationshipType().contentEquals(
row.getCell(CARDINALITY).getStringCellValue())).findFirst().orElseThrow(Exception::new),
row.getCell(SOURCE_RESOURCE).getStringCellValue(),
row.getCell(SOURCE_RESOURCE_KEY).getStringCellValue()
);
} catch (Exception ex) {
LOG.error(ex);
}
return referenceStandardRelationship;
}
public ReferenceStandardField deserializeStandardFieldRow(Row row) {
return new ReferenceStandardField.Builder()
.setStandardName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.STANDARD_NAME), row))
.setDisplayName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.DISPLAY_NAME), row))
.setDefinition(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.DEFINITION), row))
.setGroups(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.GROUPS), row))
.setSimpleDataType(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SIMPLE_DATA_TYPE), row))
.setSuggestedMaxLength(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SUGGESTED_MAX_LENGTH), row))
.setSynonyms(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SYNONYM), row))
.setElementStatus(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.ELEMENT_STATUS), row))
.setBedes(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.BEDES), row))
.setCertificationLevel(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.CERTIFICATION_LEVEL), row))
.setRecordId(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.RECORD_ID), row))
.setLookupStatus(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.LOOKUP_STATUS), row))
.setLookup(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.LOOKUP), row))
.setCollection(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.COLLECTION), row))
.setSuggestedMaxPrecision(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SUGGESTED_MAX_PRECISION), row))
.setRepeatingElement(getBooleanValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.REPEATING_ELEMENT), row))
.setPropertyTypes(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.PROPERTY_TYPES), row))
.setPayloads(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.PAYLOADS), row))
.setSpanishStandardName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SPANISH_STANDARD_NAME), row))
.setStatusChangeDate(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.STATUS_CHANGE_DATE), row))
.setRevisedDate(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.REVISED_DATE), row))
.setAddedInVersion(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.ADDED_IN_VERSION), row))
.setWikiPageTitle(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.WIKI_PAGE_TITLE), row))
.setWikiPageURL(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.WIKI_PAGE_URL), row))
.setWikiPageID(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.WIKI_PAGE_ID), row))
.build();
}
public ReferenceStandardLookup deserializeStandardEnumerationRow(Row row) {
return new ReferenceStandardLookup.Builder()
.setLookupField(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_FIELD), row))
.setLookupValue(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_VALUE), row))
.setLookupDisplayName(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_DISPLAY_NAME), row))
.setDefinition(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.DEFINITION), row))
.setLookupDisplayNameSynonyms(getArrayValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_DISPLAY_NAME_SYNONYMS), row))
.setBedes(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.BEDES), row))
.setReferences(getArrayValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.REFERENCES), row))
.setElementStatus(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.ELEMENT_STATUS), row))
.setLookupId(getIntegerValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_ID), row))
.setLookupFieldId(getIntegerValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_FIELD_ID), row))
.setSpanishLookupField(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.SPANISH_LOOKUP_FIELD), row))
.setSpanishLookupValue(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.SPANISH_LOOKUP_VALUE), row))
.setStatusChangeDate(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.STATUS_CHANGE_DATE), row))
.setRevisedDate(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.REVISED_DATE), row))
.setAddedInVersion(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.ADDED_IN_VERSION), row))
.setWikiPageTitle(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.WIKI_PAGE_TITLE), row))
.setWikiPageUrl(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.WIKI_PAGE_URL), row))
.build();
}
void processResourceSheet(Sheet sheet) {
this.sheet = sheet;
}
abstract void processNumber(ReferenceStandardField field);
abstract void processStringListSingle(ReferenceStandardField field);
abstract void processString(ReferenceStandardField field);
abstract void processBoolean(ReferenceStandardField field);
abstract void processStringListMulti(ReferenceStandardField field);
abstract void processDate(ReferenceStandardField field);
abstract void processTimestamp(ReferenceStandardField field);
abstract void processCollection(ReferenceStandardField field);
abstract void generateOutput();
public void processResourceRow(Row row) {
assertTrue(getDefaultErrorMessage("sheet name was null but was expected to contain a resource name!"),
sheet != null && sheet.getSheetName() != null);
//if there's no field in the standard name column, don't process the row
if (row.getCell(getWellKnownStandardFieldIndex(STANDARD_NAME)) == null) return;
ReferenceStandardField referenceStandardField = deserializeStandardFieldRow(row);
referenceStandardField.setParentResourceName(sheet.getSheetName());
//add empty top-level resource name map
standardFieldsMap.putIfAbsent(sheet.getSheetName(), new LinkedHashMap<>());
//add a resource, standard field
standardFieldsMap.get(sheet.getSheetName()).put(referenceStandardField.getStandardName(), referenceStandardField);
//now that row has been processed, extract field type and assemble the template
switch (referenceStandardField.getSimpleDataType()) {
case NUMBER:
processNumber(referenceStandardField);
break;
case STRING_LIST_SINGLE:
processStringListSingle(referenceStandardField);
break;
case STRING:
processString(referenceStandardField);
break;
case BOOLEAN:
processBoolean(referenceStandardField);
break;
case STRING_LIST_MULTI:
processStringListMulti(referenceStandardField);
break;
case DATE:
processDate(referenceStandardField);
break;
case TIMESTAMP:
processTimestamp(referenceStandardField);
break;
case COLLECTION:
processCollection(referenceStandardField);
break;
default:
if (referenceStandardField.getSimpleDataType() != null)
LOG.debug("Data type: " + referenceStandardField.getSimpleDataType() + " is not supported!");
}
}
String getDirectoryName() {
return startTimestamp + "-" + getReferenceResource()
.toLowerCase().substring(0, getReferenceResource().lastIndexOf("."));
}
public String getReferenceResource() {
return referenceDocument;
}
public void setReferenceResource(String referenceResource) {
this.referenceDocument = referenceResource;
}
public Workbook getReferenceWorkbook() {
try {
return new XSSFWorkbook(Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(getReferenceResource())));
} catch (Exception ex) {
LOG.error(getDefaultErrorMessage(ex));
}
return null;
}
public void beforeResourceSheetProcessed(Sheet sheet) {
//Add any before events here
}
public void afterResourceSheetProcessed(Sheet sheet) {
resourceTemplates.put(sheet.getSheetName(), markup.toString());
resetMarkupBuffer();
}
public void buildEnumerationMap() {
Sheet sheet = getReferenceWorkbook().getSheet(LOOKUP_FIELDS_AND_VALUES);
buildWellKnownStandardEnumerationHeaderMap(sheet);
AtomicReference<ReferenceStandardLookup> standardEnumeration = new AtomicReference<>();
sheet.rowIterator().forEachRemaining(row -> {
if (row.getRowNum() > 0) {
standardEnumeration.set(deserializeStandardEnumerationRow(row));
if (!standardEnumerationsMap.containsKey(standardEnumeration.get().getLookupField())) {
standardEnumerationsMap.put(standardEnumeration.get().getLookupField(), new LinkedHashSet<>());
public static Integer getIntegerValue(Integer index, Row row, Integer defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
Integer value;
DataFormatter formatter = new DataFormatter();
try {
value = Integer.parseInt(formatter.formatCellValue(row.getCell(index)));
} catch (Exception ex) {
value = defaultValue;
}
standardEnumerationsMap.get(standardEnumeration.get().getLookupField()).add(standardEnumeration.get());
}
});
}
//TODO: convert to parallel stream
public void buildStandardRelationships(Sheet worksheet) {
int FIRST_ROW_INDEX = 1;
Row currentRow;
for(int rowIndex = FIRST_ROW_INDEX; rowIndex < worksheet.getPhysicalNumberOfRows(); rowIndex++) {
currentRow = worksheet.getRow(rowIndex);
if (currentRow.getCell(TARGET_RESOURCE, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).getStringCellValue().length() > 0
&& !currentRow.getCell(TARGET_RESOURCE_KEY, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).getStringCellValue().toLowerCase().contains("keynumeric")
&& !currentRow.getCell(SOURCE_RESOURCE_KEY, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).getStringCellValue().toLowerCase().contains("keynumeric")) {
referenceStandardRelationships.add(deserializeStandardRelationshipRow(currentRow));
}
return value;
}
}
public Map<String, Set<ReferenceStandardLookup>> getEnumerations() {
return standardEnumerationsMap;
}
public static Integer getIntegerValue(Integer index, Row row) {
return getIntegerValue(index, row, null);
}
public void resetMarkupBuffer() {
markup = new StringBuffer();
}
public static String getStringValue(Integer index, Row row, String defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
String value;
DataFormatter formatter = new DataFormatter();
try {
value = formatter.formatCellValue(row.getCell(index));
} catch (Exception ex) {
value = defaultValue;
}
return value;
}
public static final class WELL_KNOWN_DATA_TYPES {
public static final String
NUMBER = "Number",
STRING_LIST_SINGLE = "String List, Single",
STRING_LIST_MULTI = "String List, Multi",
STRING = "String",
BOOLEAN = "Boolean",
DATE = "Date",
TIMESTAMP = "Timestamp",
COLLECTION = "Collection";
}
public static String getStringValue(Integer index, Row row) {
return getStringValue(index, row, null);
}
/**
* Must match what's in the DD spreadsheet EXACTLY
*/
public static final class WELL_KNOWN_FIELD_HEADERS {
public static final String
STANDARD_NAME = "StandardName",
DISPLAY_NAME = "DisplayName",
DEFINITION = "Definition",
GROUPS = "Groups",
SIMPLE_DATA_TYPE = "SimpleDataType",
SUGGESTED_MAX_LENGTH = "SugMaxLength",
SYNONYM = "Synonym",
ELEMENT_STATUS = "ElementStatus",
BEDES = "BEDES",
CERTIFICATION_LEVEL = "CertificationLevel",
RECORD_ID = "RecordID",
LOOKUP_STATUS = "LookupStatus",
LOOKUP = "Lookup",
COLLECTION = "Collection",
SUGGESTED_MAX_PRECISION = "SugMaxPrecision",
REPEATING_ELEMENT = "RepeatingElement",
PROPERTY_TYPES = "PropertyTypes",
PAYLOADS = "Payloads",
SPANISH_STANDARD_NAME = "SpanishStandardName",
STATUS_CHANGE_DATE = "StatusChangeDate",
REVISED_DATE = "RevisedDate",
ADDED_IN_VERSION = "AddedInVersion",
WIKI_PAGE_TITLE = "Wiki Page Title",
WIKI_PAGE_URL = "Wiki Page URL",
WIKI_PAGE_ID = "Wiki Page ID";
}
public static Boolean getBooleanValue(Integer index, Row row, Boolean defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
final String BOOLEAN_VALUE = "yes";
/**
* Must match what's in the DD spreadsheet EXACTLY
*/
public static final class WELL_KNOWN_ENUMERATION_HEADERS {
public static final String
LOOKUP_FIELD = "LookupField",
LOOKUP_VALUE = "LookupValue",
LOOKUP_DISPLAY_NAME = "LookupDisplayName",
DEFINITION = "Definition",
LOOKUP_DISPLAY_NAME_SYNONYMS = "LookupDisplayNameSynonyms",
BEDES = "BEDES",
REFERENCES = "References",
ELEMENT_STATUS = "ElementStatus",
LOOKUP_ID = "LookupID",
LOOKUP_FIELD_ID = "LookupFieldID",
SPANISH_LOOKUP_FIELD = "SpanishLookupField",
SPANISH_LOOKUP_VALUE = "SpanishLookupValue",
STATUS_CHANGE_DATE = "StatusChangeDate",
REVISED_DATE = "RevisedDate",
ADDED_IN_VERSION = "AddedInVersion",
WIKI_PAGE_TITLE = "Wiki Page Title",
WIKI_PAGE_URL = "Wiki Page URL";
}
Boolean value = false;
String cellValue;
DataFormatter formatter = new DataFormatter();
try {
cellValue = formatter.formatCellValue(row.getCell(index));
if (cellValue.toLowerCase().contains(BOOLEAN_VALUE)) {
value = true;
}
} catch (Exception ex) {
value = defaultValue;
}
return value;
}
public static Boolean getBooleanValue(Integer index, Row row) {
return getBooleanValue(index, row, false);
}
public static List<String> getArrayValue(Integer index, Row row, List<String> defaultValue) {
if (index == null || !(index >= 0)) return defaultValue;
DataFormatter formatter = new DataFormatter();
String cellValue;
List<String> value = new ArrayList<>();
try {
cellValue = formatter.formatCellValue(row.getCell(index));
if (cellValue != null && cellValue.length() > 0) {
//LOG.info("Cell index is: " + index + ", cell value is: " + cellValue);
value = Arrays.stream(cellValue
.replace(" ", "").split(","))
.map(String::trim)
.filter(item -> item.length() > 0)
.collect(Collectors.toList());
}
} catch (Exception ex) {
value = defaultValue;
}
return value;
}
public static List<String> getArrayValue(Integer index, Row row) {
return getArrayValue(index, row, new ArrayList<>());
}
public Integer getWellKnownStandardFieldIndex(String wellKnownStandardFieldKey) {
return wellKnownStandardFieldHeaderMap.get(wellKnownStandardFieldKey);
}
public Integer getWellKnownStandardEnumerationIndex(String wellKnownStandardEnumerationKey) {
return wellKnownStandardEnumerationHeaderMap.get(wellKnownStandardEnumerationKey);
}
public void buildWellKnownStandardEnumerationHeaderMap(Sheet sheet) {
wellKnownStandardEnumerationHeaderMap = new LinkedHashMap<>();
sheet.getRow(0).cellIterator().forEachRemaining(cell ->
wellKnownStandardEnumerationHeaderMap.put(cell.getStringCellValue(), cell.getColumnIndex()));
}
public List<ReferenceStandardRelationship> getStandardRelationships() {
return this.referenceStandardRelationships;
}
public ReferenceStandardField deserializeStandardFieldRow(Row row) {
return new ReferenceStandardField.Builder()
.setResourceName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.RESOURCE_NAME), row))
.setStandardName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.STANDARD_NAME), row))
.setDisplayName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.DISPLAY_NAME), row))
.setDefinition(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.DEFINITION), row))
.setGroups(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.GROUPS), row))
.setSimpleDataType(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SIMPLE_DATA_TYPE), row))
.setSuggestedMaxLength(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SUGGESTED_MAX_LENGTH), row))
.setSuggestedMaxPrecision(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SUGGESTED_MAX_PRECISION), row))
.setSynonyms(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SYNONYMS), row))
.setElementStatus(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.ELEMENT_STATUS), row))
.setBedes(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.BEDES), row))
.setRecordId(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.RECORD_ID), row))
.setLookupStatus(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.LOOKUP_STATUS), row))
.setLookupName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.LOOKUP_NAME), row))
.setRepeatingElement(getBooleanValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.REPEATING_ELEMENT), row))
.setPropertyTypes(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.PROPERTY_TYPES), row))
.setPayloads(getArrayValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.PAYLOADS), row))
.setSpanishDisplayName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.SPANISH_DISPLAY_NAME), row))
.setFrenchCanadianDisplayName(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.FRENCH_CANADIAN_DISPLAY_NAME), row))
.setStatusChangeDate(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.STATUS_CHANGE_DATE), row))
.setRevisedDate(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.REVISED_DATE), row))
.setAddedInVersion(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.ADDED_IN_VERSION), row))
.setWikiPageTitle(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.WIKI_PAGE_TITLE), row))
.setWikiPageURL(getStringValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.WIKI_PAGE_URL), row))
.setWikiPageID(getIntegerValue(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.WIKI_PAGE_ID), row))
.build();
}
public ReferenceStandardLookup deserializeStandardEnumerationRow(Row row) {
return new ReferenceStandardLookup.Builder()
.setLookupField(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_FIELD), row))
.setLookupValue(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_VALUE), row))
.setLookupDisplayName(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_DISPLAY_NAME), row))
.setDefinition(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.DEFINITION), row))
.setLookupDisplayNameSynonyms(getArrayValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_DISPLAY_NAME_SYNONYMS), row))
.setBedes(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.BEDES), row))
.setReferences(getArrayValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.REFERENCES), row))
.setElementStatus(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.ELEMENT_STATUS), row))
.setLookupId(getIntegerValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_ID), row))
.setLookupFieldId(getIntegerValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.LOOKUP_FIELD_ID), row))
.setSpanishLookupField(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.SPANISH_LOOKUP_FIELD), row))
.setSpanishLookupValue(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.SPANISH_LOOKUP_VALUE), row))
.setStatusChangeDate(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.STATUS_CHANGE_DATE), row))
.setRevisedDate(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.REVISED_DATE), row))
.setAddedInVersion(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.ADDED_IN_VERSION), row))
.setWikiPageTitle(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.WIKI_PAGE_TITLE), row))
.setWikiPageUrl(getStringValue(getWellKnownStandardEnumerationIndex(WELL_KNOWN_ENUMERATION_HEADERS.WIKI_PAGE_URL), row))
.build();
}
void processResourceSheet(Sheet sheet) {
this.sheet = sheet;
}
abstract void processNumber(ReferenceStandardField field);
abstract void processStringListSingle(ReferenceStandardField field);
abstract void processString(ReferenceStandardField field);
abstract void processBoolean(ReferenceStandardField field);
abstract void processStringListMulti(ReferenceStandardField field);
abstract void processDate(ReferenceStandardField field);
abstract void processTimestamp(ReferenceStandardField field);
abstract void processCollection(ReferenceStandardField field);
abstract void generateOutput();
public void processResourceRow(Row row) {
final Cell
resourceNameCell = row.getCell(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.RESOURCE_NAME)),
standardNameCell = row.getCell(getWellKnownStandardFieldIndex(WELL_KNOWN_FIELD_HEADERS.STANDARD_NAME));
if (resourceNameCell == null || resourceNameCell.getStringCellValue() == null) {
LOG.error("ResourceName value is missing for row: " + row.getRowNum());
exit(0);
}
if (standardNameCell == null || standardNameCell.getStringCellValue() == null) {
LOG.error("StandardName value is missing for row: " + row.getRowNum());
exit(0);
}
ReferenceStandardField referenceStandardField = deserializeStandardFieldRow(row);
//add empty top-level resource name map
standardFieldsMap.putIfAbsent(resourceNameCell.getStringCellValue(), new LinkedHashMap<>());
//add a resource, standard field
standardFieldsMap.get(sheet.getSheetName()).put(referenceStandardField.getStandardName(), referenceStandardField);
//now that row has been processed, extract field type and assemble the template
switch (referenceStandardField.getSimpleDataType()) {
case NUMBER:
processNumber(referenceStandardField);
break;
case STRING_LIST_SINGLE:
processStringListSingle(referenceStandardField);
break;
case STRING:
processString(referenceStandardField);
break;
case BOOLEAN:
processBoolean(referenceStandardField);
break;
case STRING_LIST_MULTI:
processStringListMulti(referenceStandardField);
break;
case DATE:
processDate(referenceStandardField);
break;
case TIMESTAMP:
processTimestamp(referenceStandardField);
break;
case COLLECTION:
processCollection(referenceStandardField);
break;
default:
if (referenceStandardField.getSimpleDataType() != null)
LOG.debug("Data type: " + referenceStandardField.getSimpleDataType() + " is not supported!");
}
}
String getDirectoryName() {
return startTimestamp + "-" + getReferenceResource()
.toLowerCase().substring(0, getReferenceResource().lastIndexOf("."));
}
public String getReferenceResource() {
return referenceDocument;
}
public void setReferenceResource(String referenceResource) {
this.referenceDocument = referenceResource;
}
public Workbook getReferenceWorkbook() {
try {
return new XSSFWorkbook(Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(getReferenceResource())));
} catch (Exception ex) {
LOG.error(getDefaultErrorMessage(ex));
}
return null;
}
public void beforeResourceSheetProcessed(Sheet sheet) {
//Add any before events here
}
public void afterResourceSheetProcessed(Sheet sheet) {
resourceTemplates.put(sheet.getSheetName(), markup.toString());
resetMarkupBuffer();
}
public void buildEnumerationMap() {
Sheet sheet = getReferenceWorkbook().getSheet(LOOKUP_FIELDS_AND_VALUES);
buildWellKnownStandardEnumerationHeaderMap(sheet);
AtomicReference<ReferenceStandardLookup> standardEnumeration = new AtomicReference<>();
sheet.rowIterator().forEachRemaining(row -> {
if (row.getRowNum() > 0) {
standardEnumeration.set(deserializeStandardEnumerationRow(row));
if (!standardEnumerationsMap.containsKey(standardEnumeration.get().getLookupField())) {
standardEnumerationsMap.put(standardEnumeration.get().getLookupField(), new LinkedHashSet<>());
}
standardEnumerationsMap.get(standardEnumeration.get().getLookupField()).add(standardEnumeration.get());
}
});
}
public Map<String, Set<ReferenceStandardLookup>> getEnumerations() {
return standardEnumerationsMap;
}
public void resetMarkupBuffer() {
markup = new StringBuffer();
}
public static final class WELL_KNOWN_DATA_TYPES {
public static final String
NUMBER = "Number",
STRING_LIST_SINGLE = "String List, Single",
STRING_LIST_MULTI = "String List, Multi",
STRING = "String",
BOOLEAN = "Boolean",
DATE = "Date",
TIMESTAMP = "Timestamp",
COLLECTION = "Collection";
}
/**
* Must match what's in the DD spreadsheet EXACTLY
*/
public static final class WELL_KNOWN_FIELD_HEADERS {
public static final String
RESOURCE_NAME = "ResourceName",
STANDARD_NAME = "StandardName",
DISPLAY_NAME = "DisplayName",
DEFINITION = "Definition",
GROUPS = "Groups",
SIMPLE_DATA_TYPE = "SimpleDataType",
SUGGESTED_MAX_LENGTH = "SugMaxLength",
SUGGESTED_MAX_PRECISION = "SugMaxPrecision",
SYNONYMS = "Synonyms",
FRENCH_CANADIAN_DISPLAY_NAME = "FrenchCanadianDisplayName",
SPANISH_DISPLAY_NAME = "SpanishDisplayName",
ELEMENT_STATUS = "ElementStatus",
RECORD_ID = "RecordId",
LOOKUP_STATUS = "LookupStatus",
LOOKUP_NAME = "LookupName",
REPEATING_ELEMENT = "RepeatingElement",
PROPERTY_TYPES = "PropertyTypes",
PAYLOADS = "Payloads",
STATUS_CHANGE_DATE = "StatusChangeDate",
REVISED_DATE = "RevisedDate",
ADDED_IN_VERSION = "AddedInVersion",
WIKI_PAGE_TITLE = "WikiPageTitle",
WIKI_PAGE_URL = "WikiPageUrl",
WIKI_PAGE_ID = "WikiPageId",
BEDES = "BEDES";
}
/**
* Must match what's in the DD spreadsheet EXACTLY
*/
public static final class WELL_KNOWN_ENUMERATION_HEADERS {
public static final String
LOOKUP_FIELD = "LookupField",
LOOKUP_VALUE = "LookupValue",
LOOKUP_DISPLAY_NAME = "LookupDisplayName",
DEFINITION = "Definition",
LOOKUP_DISPLAY_NAME_SYNONYMS = "LookupDisplayNameSynonyms",
BEDES = "BEDES",
REFERENCES = "References",
ELEMENT_STATUS = "ElementStatus",
LOOKUP_ID = "LookupID",
LOOKUP_FIELD_ID = "LookupFieldID",
SPANISH_LOOKUP_FIELD = "SpanishLookupField",
SPANISH_LOOKUP_VALUE = "SpanishLookupValue",
STATUS_CHANGE_DATE = "StatusChangeDate",
REVISED_DATE = "RevisedDate",
ADDED_IN_VERSION = "AddedInVersion",
WIKI_PAGE_TITLE = "Wiki Page Title",
WIKI_PAGE_URL = "Wiki Page URL";
}
}

View File

@ -151,11 +151,11 @@ public class LookupResource {
final ArrayList<String> fieldsWithMissingAnnotations = new ArrayList<>();
lookupFields.forEach(referenceStandardField -> {
LOG.debug("Standard Field: { "
+ "resourceName: \"" + referenceStandardField.getParentResourceName() + "\""
+ "resourceName: \"" + referenceStandardField.getResourceName() + "\""
+ ", standardName: \"" + referenceStandardField.getStandardName() + "\""
+ ", lookupName: \"" + referenceStandardField.getLookupName() + "\" }");
EdmElement foundElement = getEdmElement(container.get().getEdm(), referenceStandardField.getParentResourceName(), referenceStandardField.getStandardName());
EdmElement foundElement = getEdmElement(container.get().getEdm(), referenceStandardField.getResourceName(), referenceStandardField.getStandardName());
final boolean isStringDataType = foundElement != null &&
foundElement.getType().getFullQualifiedName().toString().contentEquals(EdmPrimitiveTypeKind.String.getFullQualifiedName().toString());

View File

@ -3,35 +3,38 @@ package org.reso.models;
import java.util.List;
/**
* Defines a standard field according to the Data Dictionary specification
* Matches the format of the Data Dictionary worksheet:
* @see <a href="https://docs.google.com/spreadsheets/d/1_59Iqr7AQ51rEFa7p0ND-YhJjEru8gY-D_HM1yy5c6w/edit#gid=544946974" />
*
*/
public class ReferenceStandardField {
private String resourceName;
private String standardName;
private String displayName;
private String parentResourceName;
private String definition;
private List<String> groups;
private String simpleDataType;
private String sourceResource;
private Integer suggestedMaxLength;
private List<String> synonyms;
private String elementStatus;
private String bedes;
private String certificationLevel;
private Integer recordId;
private String lookupStatus;
private String lookup;
private String collection;
private Integer suggestedMaxPrecision;
private List<String> synonyms;
private String frenchCanadianDisplayName;
private String spanishDisplayName;
private String elementStatus;
private String recordId;
private String lookupStatus;
private String lookupName;
private Boolean repeatingElement;
private List<String> propertyTypes;
private List<String> payloads;
private String spanishStandardName;
private String statusChangeDate;
private String revisedDate;
private String addedInVersion;
private String wikiPageTitle;
private String wikiPageUrl;
private Integer wikiPageId;
private String bedes;
private ReferenceStandardField() { /* private constructor use Builder instead */ }
@ -54,8 +57,8 @@ public class ReferenceStandardField {
return displayName;
}
public String getParentResourceName(){
return parentResourceName;
public String getResourceName(){
return resourceName;
}
public String getDefinition() {
@ -86,11 +89,7 @@ public class ReferenceStandardField {
return bedes;
}
public String getCertificationLevel() {
return certificationLevel;
}
public Integer getRecordId() {
public String getRecordId() {
return recordId;
}
@ -98,22 +97,16 @@ public class ReferenceStandardField {
return lookupStatus;
}
public String getLookup() {
return lookup;
}
public String getLookupName() {
String lookupName = getLookup()
String parsedLookupName = lookupName
.replace("<n/a>", "")
.replace("Lookups", "").trim();
if (lookupName.length() == 0) return null;
return lookupName;
}
public String getCollection() {
return collection;
if (parsedLookupName.length() == 0) {
return null;
} else {
return parsedLookupName;
}
}
public Integer getSuggestedMaxPrecision() {
@ -132,8 +125,12 @@ public class ReferenceStandardField {
return payloads;
}
public String getSpanishStandardName() {
return spanishStandardName;
public String getSpanishDisplayName() {
return spanishDisplayName;
}
public String getFrenchCanadianDisplayName() {
return frenchCanadianDisplayName;
}
public String getStatusChangeDate() {
@ -160,8 +157,8 @@ public class ReferenceStandardField {
return wikiPageId;
}
public void setParentResourceName(String parentResourceName) {
this.parentResourceName = parentResourceName;
public void setResourceName() {
this.resourceName = resourceName;
}
public static class Builder {
@ -177,8 +174,8 @@ public class ReferenceStandardField {
return this;
}
public Builder setParentResourceName(String parentResourceName) {
referenceStandardField.parentResourceName = parentResourceName;
public Builder setResourceName(String resourceName) {
referenceStandardField.resourceName = resourceName;
return this;
}
@ -217,18 +214,13 @@ public class ReferenceStandardField {
return this;
}
public Builder setCertificationLevel(String certificationLevel) {
referenceStandardField.certificationLevel = certificationLevel;
return this;
}
public Builder setRecordId(Integer recordId) {
public Builder setRecordId(String recordId) {
referenceStandardField.recordId = recordId;
return this;
}
public Builder setLookup(String lookup) {
referenceStandardField.lookup = lookup;
public Builder setLookupName(String lookupName) {
referenceStandardField.lookupName = lookupName;
return this;
}
@ -237,11 +229,6 @@ public class ReferenceStandardField {
return this;
}
public Builder setCollection(String collection) {
referenceStandardField.collection = collection;
return this;
}
public Builder setSuggestedMaxPrecision(Integer suggestedMaxPrecision) {
referenceStandardField.suggestedMaxPrecision = suggestedMaxPrecision;
return this;
@ -262,8 +249,8 @@ public class ReferenceStandardField {
return this;
}
public Builder setSpanishStandardName(String spanishStandardName) {
referenceStandardField.spanishStandardName = spanishStandardName;
public Builder setSpanishDisplayName(String spanishDisplayName) {
referenceStandardField.spanishDisplayName = spanishDisplayName;
return this;
}
@ -297,6 +284,16 @@ public class ReferenceStandardField {
return this;
}
public Builder setSourceResource(String sourceResource) {
referenceStandardField.sourceResource = sourceResource;
return this;
}
public Builder setFrenchCanadianDisplayName(String frenchCanadianDisplayName) {
referenceStandardField.frenchCanadianDisplayName = frenchCanadianDisplayName;
return this;
}
public ReferenceStandardField build() {
return referenceStandardField;
}