From 856c7bc0375272328cf65fda8df7bf3c423ec76a Mon Sep 17 00:00:00 2001 From: Joshua Darnell Date: Thu, 22 Jul 2021 05:05:34 -0700 Subject: [PATCH] Issue #82: refactored availability report to use frequencies and track lookups at a granular level (#83) --- .../codegen/DDCacheProcessor.java | 17 +- .../codegen/DataDictionaryCodeGenerator.java | 2 + .../codegen/WorksheetProcessor.java | 1 + .../stepdefs/DataAvailability.java | 81 +++++- .../java/org/reso/models/LookupValue.java | 42 +++ .../org/reso/models/PayloadSampleReport.java | 270 +++++++++++++----- 6 files changed, 310 insertions(+), 103 deletions(-) create mode 100644 src/main/java/org/reso/models/LookupValue.java diff --git a/src/main/java/org/reso/certification/codegen/DDCacheProcessor.java b/src/main/java/org/reso/certification/codegen/DDCacheProcessor.java index 6558b04..46a0a9b 100644 --- a/src/main/java/org/reso/certification/codegen/DDCacheProcessor.java +++ b/src/main/java/org/reso/certification/codegen/DDCacheProcessor.java @@ -2,21 +2,20 @@ package org.reso.certification.codegen; import org.reso.models.ReferenceStandardField; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; public class DDCacheProcessor extends WorksheetProcessor { - Map> standardFieldCache = new LinkedHashMap<>(); + final AtomicReference>> standardFieldCache = + new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); private void addToFieldCache(ReferenceStandardField field) { - standardFieldCache.putIfAbsent(field.getParentResourceName(), new LinkedList<>()); - standardFieldCache.get(field.getParentResourceName()).add(field); + standardFieldCache.get().putIfAbsent(field.getParentResourceName(), new LinkedHashMap<>()); + standardFieldCache.get().get(field.getParentResourceName()).put(field.getStandardName(), field); } - public Map> getStandardFieldCache() { - return standardFieldCache; + public Map> getStandardFieldCache() { + return standardFieldCache.get(); } @Override diff --git a/src/main/java/org/reso/certification/codegen/DataDictionaryCodeGenerator.java b/src/main/java/org/reso/certification/codegen/DataDictionaryCodeGenerator.java index 5a52d68..bca7d13 100644 --- a/src/main/java/org/reso/certification/codegen/DataDictionaryCodeGenerator.java +++ b/src/main/java/org/reso/certification/codegen/DataDictionaryCodeGenerator.java @@ -31,6 +31,8 @@ public class DataDictionaryCodeGenerator { /** * Generates Data Dictionary references for local workbook instance using the configured WorksheetProcessor + * + * TODO: convert to .parallelStream() */ public void processWorksheets() { Sheet currentWorksheet, standardResourcesWorksheet; diff --git a/src/main/java/org/reso/certification/codegen/WorksheetProcessor.java b/src/main/java/org/reso/certification/codegen/WorksheetProcessor.java index 9b1b2e6..dd5ca0e 100644 --- a/src/main/java/org/reso/certification/codegen/WorksheetProcessor.java +++ b/src/main/java/org/reso/certification/codegen/WorksheetProcessor.java @@ -353,6 +353,7 @@ public abstract class WorksheetProcessor { //enumerations.forEach((key, items) -> LOG.info("key: " + key + " , items: " + items.toString())); } + //TODO: convert to parallel stream public void buildStandardRelationships(Sheet worksheet) { int FIRST_ROW_INDEX = 1; Row currentRow; diff --git a/src/main/java/org/reso/certification/stepdefs/DataAvailability.java b/src/main/java/org/reso/certification/stepdefs/DataAvailability.java index a7346bd..fa89b0a 100644 --- a/src/main/java/org/reso/certification/stepdefs/DataAvailability.java +++ b/src/main/java/org/reso/certification/stepdefs/DataAvailability.java @@ -46,6 +46,8 @@ import java.util.stream.Collectors; import static io.restassured.path.json.JsonPath.from; import static org.junit.Assert.assertNotNull; import static org.junit.Assume.assumeTrue; +import static org.reso.certification.codegen.WorksheetProcessor.WELL_KNOWN_DATA_TYPES.STRING_LIST_MULTI; +import static org.reso.certification.codegen.WorksheetProcessor.WELL_KNOWN_DATA_TYPES.STRING_LIST_SINGLE; import static org.reso.certification.containers.WebAPITestContainer.EMPTY_STRING; import static org.reso.commander.Commander.NOT_OK; import static org.reso.commander.common.ErrorMsg.getDefaultErrorMessage; @@ -80,15 +82,22 @@ public class DataAvailability { private final static AtomicReference container = new AtomicReference<>(); private final static AtomicBoolean hasSamplesDirectoryBeenCleared = new AtomicBoolean(false); + //TODO: compute moving averages and search each payload sample immediately so no collection is needed private final static AtomicReference>> resourcePayloadSampleMap = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); - private final static AtomicReference>> standardFieldCache = + private final static AtomicReference>> resourceFieldMap = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); private final static AtomicReference> resourceCounts = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); + //resourceName, fieldName, lookupName, lookupValue, tally + private final static AtomicReference> resourceFieldLookupTallies = + new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); + + private final static AtomicReference processor = new AtomicReference<>(); + @Inject public DataAvailability(WebAPITestContainer c) { container.set(c); @@ -110,12 +119,15 @@ public class DataAvailability { /** * Creates a data availability report for the given samples map + * * @param resourcePayloadSamplesMap the samples map to create the report from - * @param reportName the name of the report + * @param reportName the name of the report */ - public void createDataAvailabilityReport(Map> resourcePayloadSamplesMap, - String reportName, Map resourceCounts) { - PayloadSampleReport payloadSampleReport = new PayloadSampleReport(container.get().getEdm(), resourcePayloadSamplesMap, resourceCounts); + public void createDataAvailabilityReport(Map> resourcePayloadSamplesMap, String reportName, + Map resourceCounts, Map resourceFieldLookupTallies) { + + PayloadSampleReport payloadSampleReport = + new PayloadSampleReport(container.get().getEdm(), resourcePayloadSamplesMap, resourceCounts, resourceFieldLookupTallies); GsonBuilder gsonBuilder = new GsonBuilder().setPrettyPrinting(); gsonBuilder.registerTypeAdapter(PayloadSampleReport.class, payloadSampleReport); @@ -129,14 +141,16 @@ public class DataAvailability { * @return the SHA hash of the given values */ private static String hashValues(String... values) { + //noinspection UnstableApiUsage return Hashing.sha256().hashString(String.join(EMPTY_STRING, values), StandardCharsets.UTF_8).toString(); } /** * Builds a request URI string, taking into account whether the sampling is being done with an optional * filter, for instance in the shared systems case - * @param resourceName the resource name to query - * @param timestampField the timestamp field for the resource + * + * @param resourceName the resource name to query + * @param timestampField the timestamp field for the resource * @param lastFetchedDate the last fetched date for filtering * @return a string OData query used for sampling */ @@ -154,6 +168,7 @@ public class DataAvailability { /** * Builds a request URI string for counting the number of available items on a resource, taking into account * whether the sample is being done with an optional filter, for instance in the shared system case + * * @param resourceName the resource name to query * @return a request URI string for getting OData counts */ @@ -169,6 +184,7 @@ public class DataAvailability { /** * Queries the server and fetches a resource count for the given resource name + * * @param resourceName the resource name to get the count for * @return the count found for the resource, or null if the request did not return a count */ @@ -227,7 +243,7 @@ public class DataAvailability { edmSchema.getEntityTypes().stream().filter(edmEntityType -> edmEntityType.getName().equals(resourceName)) .findFirst().ifPresent(entityType::set)); - //return null if the entity type isn't defined + //return an empty list if the entity type isn't defined if (entityType.get() == null) return new ArrayList<>(); if (entityType.get().getProperty(MODIFICATION_TIMESTAMP_FIELD) == null) { @@ -307,6 +323,7 @@ public class DataAvailability { } break; } else { + //TODO: add pluralizer LOG.info("Time taken: " + (transportWrapper.get().getElapsedTimeMillis() >= 1000 ? (transportWrapper.get().getElapsedTimeMillis() / 1000) + "s" : transportWrapper.get().getElapsedTimeMillis() + "ms")); @@ -350,6 +367,41 @@ public class DataAvailability { } } + + //if the field is a lookup field, collect the frequency of each unique set of enumerations for the field + if (property.isEnum() || (processor.get().getStandardFieldCache().containsKey(resourceName) + && processor.get().getStandardFieldCache().get(resourceName).containsKey(property.getName()))) { + ReferenceStandardField standardField = processor.get().getStandardFieldCache().get(resourceName).get(property.getName()); + //if the field is declared as an OData Edm.EnumType or String List, Single or Multii in the DD, then collect its value + if (property.isEnum() || (standardField.getSimpleDataType().contentEquals(STRING_LIST_SINGLE) + || standardField.getSimpleDataType().contentEquals(STRING_LIST_MULTI))) { + + ArrayList values = new ArrayList<>(); + + if (value == null || value.contentEquals("[]")) { + values.add("null"); + } else { + if (property.isCollection()) { + property.asCollection().forEach(v -> values.add(v.toString())); + } else { + if (value.contains(",")) { + values.addAll(Arrays.asList(value.split(","))); + } else { + values.add(value); + } + } + } + + values.forEach(v -> { + LookupValue binder = new LookupValue(resourceName, property.getName(), v); + resourceFieldLookupTallies.get().putIfAbsent(binder, 0); + + //now increment the lookup value + resourceFieldLookupTallies.get().put(binder, resourceFieldLookupTallies.get().get(binder) + 1); + }); + } + } + //turn off hashing when DEBUG is true if (!DEBUG && value != null) { if (!(property.getName().contentEquals(timestampField.get()) @@ -418,12 +470,11 @@ public class DataAvailability { public void thatValidMetadataHaveBeenRequestedFromTheServer() { try { if (container.get().hasValidMetadata()) { - if (standardFieldCache.get().size() == 0) { + if (processor.get() == null || processor.get().getStandardFieldCache().size() == 0) { LOG.info("Creating standard field cache..."); - DDCacheProcessor cacheProcessor = new DDCacheProcessor(); - DataDictionaryCodeGenerator generator = new DataDictionaryCodeGenerator(cacheProcessor); + processor.set(new DDCacheProcessor()); + DataDictionaryCodeGenerator generator = new DataDictionaryCodeGenerator(processor.get()); generator.processWorksheets(); - standardFieldCache.get().putAll(cacheProcessor.getStandardFieldCache()); LOG.info("Standard field cache created!"); } } else { @@ -437,8 +488,8 @@ public class DataAvailability { @And("the metadata contains RESO Standard Resources") public void theMetadataContainsRESOStandardResources() { Set resources = container.get().getEdm().getSchemas().stream().map(schema -> - schema.getEntityTypes().stream().map(EdmNamed::getName) - .collect(Collectors.toSet())) + schema.getEntityTypes().stream().map(EdmNamed::getName) + .collect(Collectors.toSet())) .flatMap(Collection::stream) .collect(Collectors.toSet()); @@ -535,7 +586,7 @@ public class DataAvailability { assumeTrue(true); } LOG.info("\n\nCreating data availability report!"); - createDataAvailabilityReport(resourcePayloadSampleMap.get(), reportFileName, resourceCounts.get()); + createDataAvailabilityReport(resourcePayloadSampleMap.get(), reportFileName, resourceCounts.get(), resourceFieldLookupTallies.get()); } @And("{string} has been created in the build directory") diff --git a/src/main/java/org/reso/models/LookupValue.java b/src/main/java/org/reso/models/LookupValue.java new file mode 100644 index 0000000..0349fd0 --- /dev/null +++ b/src/main/java/org/reso/models/LookupValue.java @@ -0,0 +1,42 @@ +package org.reso.models; + +import java.util.Objects; + +public final class LookupValue { + private final String resourceName; + private final String fieldName; + private final String lookupValue; + + public LookupValue(String resourceName, String fieldName, String lookupValue) { + this.resourceName = resourceName; + this.fieldName = fieldName; + this.lookupValue = lookupValue; + } + + public String getResourceName() { + return resourceName; + } + + public String getFieldName() { + return fieldName; + } + + public String getLookupValue() { + return lookupValue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LookupValue that = (LookupValue) o; + return resourceName.equals(that.resourceName) && + fieldName.equals(that.fieldName) && + lookupValue.equals(that.lookupValue); + } + + @Override + public int hashCode() { + return Objects.hash(resourceName, fieldName, lookupValue); + } +} diff --git a/src/main/java/org/reso/models/PayloadSampleReport.java b/src/main/java/org/reso/models/PayloadSampleReport.java index 12e8f3e..205241d 100644 --- a/src/main/java/org/reso/models/PayloadSampleReport.java +++ b/src/main/java/org/reso/models/PayloadSampleReport.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.olingo.commons.api.edm.Edm; import org.apache.olingo.commons.api.edm.EdmElement; +import org.apache.regexp.RE; import org.reso.commander.common.Utils; import java.lang.reflect.Type; @@ -18,31 +19,29 @@ import java.util.concurrent.atomic.AtomicReference; public class PayloadSampleReport implements JsonSerializer { private static final Logger LOG = LogManager.getLogger(PayloadSampleReport.class); private static final String POSTAL_CODE_KEY = "PostalCode"; - private final Map> resourcePayloadSamplesMap = Collections.synchronizedMap(new LinkedHashMap<>()); - private final Map> resourceFieldTallies = Collections.synchronizedMap(new LinkedHashMap<>(new LinkedHashMap<>())); - private final Map resourceCounts = Collections.synchronizedMap(new LinkedHashMap<>()); + private static final AtomicReference>> resourcePayloadSamplesMap = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); + private static final AtomicReference>> resourceFieldFrequencyMap = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>(new LinkedHashMap<>()))); + private static final AtomicReference> lookupValueFrequencyMap = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); + private static final AtomicReference> resourceCounts = new AtomicReference<>(Collections.synchronizedMap(new LinkedHashMap<>())); - private Edm metadata; + private static Edm metadata; - private PayloadSampleReport() { - //private default constructor - } - - public PayloadSampleReport(final Edm metadata, final Map> resourcePayloadSamplesMap, final Map resourceCounts) { - this.metadata = metadata; - this.resourcePayloadSamplesMap.putAll(resourcePayloadSamplesMap); - resourceFieldTallies.putAll(createResourceFieldTallies(resourcePayloadSamplesMap)); - - this.resourceCounts.putAll(resourceCounts); + public PayloadSampleReport(final Edm metadata, final Map> resourcePayloadSamplesMap, + final Map resourceCounts, final Map lookupValueFrequencyMap) { + PayloadSampleReport.metadata = metadata; + PayloadSampleReport.resourcePayloadSamplesMap.get().putAll(resourcePayloadSamplesMap); + PayloadSampleReport.resourceFieldFrequencyMap.get().putAll(createResourceFieldTallies(resourcePayloadSamplesMap)); + PayloadSampleReport.lookupValueFrequencyMap.get().putAll(lookupValueFrequencyMap); + PayloadSampleReport.resourceCounts.get().putAll(resourceCounts); } @Override public String toString() { - return String.valueOf(serialize(this, FieldAvailabilityJson.class, null)); + return String.valueOf(serialize(this, FieldsJson.class, null)); } /** - * FieldAvailabilityJson uses a JSON payload with the following structure: + * FieldsJson uses a JSON payload with the following structure: * * { * "resourceName": "Property", @@ -50,17 +49,17 @@ public class PayloadSampleReport implements JsonSerializer * "availability": 0.1 * } */ - private final class FieldAvailabilityJson implements JsonSerializer { + private static final class FieldsJson implements JsonSerializer { static final String RESOURCE_NAME_KEY = "resourceName", FIELD_NAME_KEY = "fieldName", FIELDS_KEY = "fields", - AVAILABILITY_KEY = "availability"; + FREQUENCY_KEY = "frequency"; String resourceName; EdmElement edmElement; - public FieldAvailabilityJson(String resourceName, EdmElement edmElement) { + public FieldsJson(String resourceName, EdmElement edmElement) { this.resourceName = resourceName; this.edmElement = edmElement; } @@ -72,32 +71,81 @@ public class PayloadSampleReport implements JsonSerializer reportBuilder.append(field.getAsJsonObject().get(RESOURCE_NAME_KEY)); reportBuilder.append("\nField: "); reportBuilder.append(field.getAsJsonObject().get(FIELD_NAME_KEY)); - reportBuilder.append("\nAvailability: "); - reportBuilder.append(field.getAsJsonObject().get(AVAILABILITY_KEY)); + reportBuilder.append("\nFrequency: "); + reportBuilder.append(field.getAsJsonObject().get(FREQUENCY_KEY)); reportBuilder.append("\n"); }); return reportBuilder.toString(); } @Override - public JsonElement serialize(FieldAvailabilityJson src, Type typeOfSrc, JsonSerializationContext context) { + public JsonElement serialize(FieldsJson src, Type typeOfSrc, JsonSerializationContext context) { JsonObject field = new JsonObject(); - int numTimesPresent = resourceFieldTallies.get(src.resourceName) != null - && resourceFieldTallies.get(src.resourceName).get(src.edmElement.getName()) != null - ? resourceFieldTallies.get(src.resourceName).get(src.edmElement.getName()) : 0; - - int numSamples = resourcePayloadSamplesMap.get(src.resourceName) != null - ? resourcePayloadSamplesMap.get(src.resourceName).stream().reduce(0, (a, f) -> a + f.encodedSamples.size(), Integer::sum) : 0; + int frequency = resourceFieldFrequencyMap.get().get(src.resourceName) != null + && resourceFieldFrequencyMap.get().get(src.resourceName).get(src.edmElement.getName()) != null + ? resourceFieldFrequencyMap.get().get(src.resourceName).get(src.edmElement.getName()) : 0; field.addProperty(RESOURCE_NAME_KEY, src.resourceName); field.addProperty(FIELD_NAME_KEY, src.edmElement.getName()); - field.addProperty(AVAILABILITY_KEY, numSamples > 0 ? (1.0 * numTimesPresent) / numSamples : 0); + field.addProperty(FREQUENCY_KEY, frequency); return field; } } + /** + * resourceName: "Property", + * fieldName: "StateOrProvince", + * lookupName: "StateOrProvince", + * lookupValue: "CA", + * availability: 0.03 + */ + private static final class LookupValuesJson implements JsonSerializer { + final String resourceName, fieldName, lookupValue; + final Integer frequency; + + static final String + RESOURCE_NAME_KEY = "resourceName", + FIELD_NAME_KEY = "fieldName", + LOOKUP_VALUE_KEY = "lookupValue", + FREQUENCY_KEY = "frequency"; + + public LookupValuesJson(String resourceName, String fieldName, String lookupValue, Integer frequency) { + this.resourceName = resourceName; + this.fieldName = fieldName; + this.lookupValue = lookupValue; + this.frequency = frequency; + } + + /** + * Gson invokes this call-back method during serialization when it encounters a field of the + * specified type. + * + *

In the implementation of this call-back method, you should consider invoking + * {@link JsonSerializationContext#serialize(Object, Type)} method to create JsonElements for any + * non-trivial field of the {@code src} object. However, you should never invoke it on the + * {@code src} object itself since that will cause an infinite loop (Gson will call your + * call-back method again).

+ * + * @param src the object that needs to be converted to Json. + * @param typeOfSrc the actual type (fully genericized version) of the source object. + * @param context the context of the request + * @return a JsonElement corresponding to the specified object. + */ + @Override + public JsonElement serialize(LookupValuesJson src, Type typeOfSrc, JsonSerializationContext context) { + JsonObject obj = new JsonObject(); + + obj.addProperty(RESOURCE_NAME_KEY, resourceName); + obj.addProperty(FIELD_NAME_KEY, fieldName); + obj.addProperty(LOOKUP_VALUE_KEY, lookupValue); + obj.addProperty(FREQUENCY_KEY, frequency); + + return obj; + } + } + private static Map> createResourceFieldTallies(Map> resourcePayloadSamplesMap) { AtomicReference>> resourceTallies = new AtomicReference<>(new LinkedHashMap<>()); AtomicInteger numSamples = new AtomicInteger(0); @@ -111,16 +159,15 @@ public class PayloadSampleReport implements JsonSerializer //as well as the number of samples in each case resourceTallies.get().putIfAbsent(resourceName, new LinkedHashMap<>()); if (numSamples.get() > 0) { - resourcePayloadSamplesMap.get(resourceName).forEach(payloadSample -> { - payloadSample.getSamples().forEach(sample -> { - sample.forEach((fieldName, encodedValue) -> { - if (encodedValue != null) { - resourceTallies.get().get(resourceName).putIfAbsent(fieldName, 0); - resourceTallies.get().get(resourceName).put(fieldName, resourceTallies.get().get(resourceName).get(fieldName) + 1); - } - }); - }); - }); + resourcePayloadSamplesMap.get(resourceName) + .forEach(payloadSample -> payloadSample.getSamples() + .forEach(sample -> sample + .forEach((fieldName, encodedValue) -> { + if (encodedValue != null) { + resourceTallies.get().get(resourceName).putIfAbsent(fieldName, 0); + resourceTallies.get().get(resourceName).put(fieldName, resourceTallies.get().get(resourceName).get(fieldName) + 1); + } + }))); } }); return resourceTallies.get(); @@ -133,67 +180,91 @@ public class PayloadSampleReport implements JsonSerializer VERSION_KEY = "version", VERSION = "1.7", GENERATED_ON_KEY = "generatedOn", RESOURCE_INFO_KEY = "resources", - FIELDS_KEY = "fields"; + FIELDS_KEY = "fields", + LOOKUPS_KEY = "lookups", + LOOKUP_VALUES_KEY = "lookupValues"; + //serialize fields JsonArray fields = new JsonArray(); - - src.metadata.getSchemas().forEach(edmSchema -> { + metadata.getSchemas().forEach(edmSchema -> { //serialize entities (resources) and members (fields) - edmSchema.getEntityTypes().forEach(edmEntityType -> { - edmEntityType.getPropertyNames().forEach(propertyName -> { - FieldAvailabilityJson fieldJson = new FieldAvailabilityJson(edmEntityType.getName(), edmEntityType.getProperty(propertyName)); - fields.add(fieldJson.serialize(fieldJson, FieldAvailabilityJson.class, null)); - }); + edmSchema.getEntityTypes().forEach(edmEntityType -> edmEntityType.getPropertyNames().forEach(propertyName -> { + FieldsJson fieldJson = new FieldsJson(edmEntityType.getName(), edmEntityType.getProperty(propertyName)); + fields.add(fieldJson.serialize(fieldJson, FieldsJson.class, null)); + })); + }); + + //serialize lookups + JsonArray lookups = new JsonArray(); + final Map> resourceFieldLookupTotalsMap = new LinkedHashMap<>(); + lookupValueFrequencyMap.get().forEach((lookupValue, frequency) -> { + resourceFieldLookupTotalsMap.putIfAbsent(lookupValue.getResourceName(), new LinkedHashMap<>()); + resourceFieldLookupTotalsMap.get(lookupValue.getResourceName()).putIfAbsent(lookupValue.getFieldName(), 0); + resourceFieldLookupTotalsMap.get(lookupValue.getResourceName()).put(lookupValue.getFieldName(), + resourceFieldLookupTotalsMap.get(lookupValue.getResourceName()).get(lookupValue.getFieldName()) + frequency); + }); + + resourceFieldLookupTotalsMap.forEach((resourceName, fieldLookupTotalsMap) -> { + fieldLookupTotalsMap.forEach((fieldName, numLookupsTotal) -> { + LookupsJson lookupsJson = new LookupsJson(resourceName, fieldName, numLookupsTotal); + lookups.add(lookupsJson.serialize(lookupsJson, LookupsJson.class, null)); }); }); + //serialize lookup values + JsonArray lookupValues = new JsonArray(); + lookupValueFrequencyMap.get().forEach((lookupValue, frequency) -> { + LookupValuesJson lookupValuesJson = new LookupValuesJson(lookupValue.getResourceName(), lookupValue.getFieldName(), lookupValue.getLookupValue(), frequency); + lookupValues.add(lookupValuesJson.serialize(lookupValuesJson, LookupValuesJson.class, null)); + }); + JsonObject availabilityReport = new JsonObject(); availabilityReport.addProperty(DESCRIPTION_KEY, DESCRIPTION); availabilityReport.addProperty(VERSION_KEY, VERSION); availabilityReport.addProperty(GENERATED_ON_KEY, Utils.getIsoTimestamp()); final JsonArray resourceTotalsByResource = new JsonArray(); - src.resourcePayloadSamplesMap.keySet().forEach(resourceName -> { + resourcePayloadSamplesMap.get().keySet().forEach(resourceName -> { Set postalCodes = new LinkedHashSet<>(); - ResourceInfo resourceInfo = new ResourceInfo(resourceName); + ResourcesJson resourcesJson = new ResourcesJson(resourceName); int resourceRecordCount = 0; - if (src.resourceCounts.get(resourceName) != null) { - resourceRecordCount = src.resourceCounts.get(resourceName); + if (resourceCounts.get().get(resourceName) != null) { + resourceRecordCount = resourceCounts.get().get(resourceName); } - resourceInfo.numRecordsTotal.set(resourceRecordCount); + resourcesJson.numRecordsTotal.set(resourceRecordCount); - PayloadSample zerothSample = resourcePayloadSamplesMap.get(resourceName) != null - && resourcePayloadSamplesMap.get(resourceName).size() > 0 - ? resourcePayloadSamplesMap.get(resourceName).get(0) : null; + PayloadSample zerothSample = resourcePayloadSamplesMap.get().get(resourceName) != null + && resourcePayloadSamplesMap.get().get(resourceName).size() > 0 + ? resourcePayloadSamplesMap.get().get(resourceName).get(0) : null; if (zerothSample != null) { - resourceInfo.keyFields.set(zerothSample.keyFields); - resourceInfo.dateField.set(zerothSample.dateField); + resourcesJson.keyFields.set(zerothSample.keyFields); + resourcesJson.dateField.set(zerothSample.dateField); } - if (src.resourcePayloadSamplesMap.get(resourceName) != null) { + if (resourcePayloadSamplesMap.get().get(resourceName) != null) { AtomicReference offsetDateTime = new AtomicReference<>(); - src.resourcePayloadSamplesMap.get(resourceName).forEach(payloadSample -> { - resourceInfo.totalBytesReceived.getAndAdd(payloadSample.getResponseSizeBytes()); - resourceInfo.totalResponseTimeMillis.getAndAdd(payloadSample.getResponseTimeMillis()); - resourceInfo.numSamplesProcessed.getAndIncrement(); - resourceInfo.numRecordsFetched.getAndAdd(payloadSample.encodedSamples.size()); + resourcePayloadSamplesMap.get().get(resourceName).forEach(payloadSample -> { + resourcesJson.totalBytesReceived.getAndAdd(payloadSample.getResponseSizeBytes()); + resourcesJson.totalResponseTimeMillis.getAndAdd(payloadSample.getResponseTimeMillis()); + resourcesJson.numSamplesProcessed.getAndIncrement(); + resourcesJson.numRecordsFetched.getAndAdd(payloadSample.encodedSamples.size()); payloadSample.encodedSamples.forEach(encodedSample -> { offsetDateTime.set(OffsetDateTime.parse(encodedSample.get(payloadSample.dateField))); if (offsetDateTime.get() != null) { - if (resourceInfo.dateLow.get() == null) { - resourceInfo.dateLow.set(offsetDateTime.get()); - } else if (offsetDateTime.get().isBefore(resourceInfo.dateLow.get())) { - resourceInfo.dateLow.set(offsetDateTime.get()); + if (resourcesJson.dateLow.get() == null) { + resourcesJson.dateLow.set(offsetDateTime.get()); + } else if (offsetDateTime.get().isBefore(resourcesJson.dateLow.get())) { + resourcesJson.dateLow.set(offsetDateTime.get()); } - if (resourceInfo.dateHigh.get() == null) { - resourceInfo.dateHigh.set(offsetDateTime.get()); - } else if (offsetDateTime.get().isAfter(resourceInfo.dateHigh.get())) { - resourceInfo.dateHigh.set(offsetDateTime.get()); + if (resourcesJson.dateHigh.get() == null) { + resourcesJson.dateHigh.set(offsetDateTime.get()); + } else if (offsetDateTime.get().isAfter(resourcesJson.dateHigh.get())) { + resourcesJson.dateHigh.set(offsetDateTime.get()); } } @@ -202,23 +273,65 @@ public class PayloadSampleReport implements JsonSerializer } }); - if (resourceInfo.pageSize.get() == 0) resourceInfo.pageSize.set(payloadSample.getSamples().size()); + if (resourcesJson.pageSize.get() == 0) resourcesJson.pageSize.set(payloadSample.getSamples().size()); }); } if (postalCodes.size() > 0) { - resourceInfo.postalCodes.set(postalCodes); + resourcesJson.postalCodes.set(postalCodes); } - resourceTotalsByResource.add(resourceInfo.serialize(resourceInfo, ResourceInfo.class, null)); + resourceTotalsByResource.add(resourcesJson.serialize(resourcesJson, ResourcesJson.class, null)); }); availabilityReport.add(RESOURCE_INFO_KEY, resourceTotalsByResource); availabilityReport.add(FIELDS_KEY, fields); + availabilityReport.add(LOOKUPS_KEY, lookups); + availabilityReport.add(LOOKUP_VALUES_KEY, lookupValues); return availabilityReport; } - static final class ResourceInfo implements JsonSerializer { + static final class LookupsJson implements JsonSerializer { + final String resourceName, fieldName; + final Integer numLookupsTotal; + + public LookupsJson(String resourceName, String fieldName, Integer numLookupsTotal) { + this.resourceName = resourceName; + this.fieldName = fieldName; + this.numLookupsTotal = numLookupsTotal; + } + + final String + RESOURCE_NAME_KEY = "resourceName", + FIELD_NAME_KEY = "fieldName", + NUM_LOOKUPS_TOTAL = "numLookupsTotal"; + + /** + * Gson invokes this call-back method during serialization when it encounters a field of the + * specified type. + * + *

In the implementation of this call-back method, you should consider invoking + * {@link JsonSerializationContext#serialize(Object, Type)} method to create JsonElements for any + * non-trivial field of the {@code src} object. However, you should never invoke it on the + * {@code src} object itself since that will cause an infinite loop (Gson will call your + * call-back method again).

+ * + * @param src the object that needs to be converted to Json. + * @param typeOfSrc the actual type (fully genericized version) of the source object. + * @param context the context of the request + * @return a JsonElement corresponding to the specified object. + */ + @Override + public JsonElement serialize(LookupsJson src, Type typeOfSrc, JsonSerializationContext context) { + JsonObject obj = new JsonObject(); + obj.addProperty(RESOURCE_NAME_KEY, src.resourceName); + obj.addProperty(FIELD_NAME_KEY, src.fieldName); + obj.addProperty(NUM_LOOKUPS_TOTAL, src.numLookupsTotal); + return obj; + } + } + + static final class ResourcesJson implements JsonSerializer { final String RESOURCE_NAME_KEY = "resourceName", RECORD_COUNT_KEY = "recordCount", @@ -246,7 +359,7 @@ public class PayloadSampleReport implements JsonSerializer final AtomicReference dateHigh = new AtomicReference<>(null); final AtomicReference> postalCodes = new AtomicReference<>(new LinkedHashSet<>()); - public ResourceInfo(String resourceName) { + public ResourcesJson(String resourceName) { this.resourceName.set(resourceName); } @@ -262,11 +375,11 @@ public class PayloadSampleReport implements JsonSerializer * * @param src the object that needs to be converted to Json. * @param typeOfSrc the actual type (fully genericized version) of the source object. - * @param context + * @param context the context of the request * @return a JsonElement corresponding to the specified object. */ @Override - public JsonElement serialize(ResourceInfo src, Type typeOfSrc, JsonSerializationContext context) { + public JsonElement serialize(ResourcesJson src, Type typeOfSrc, JsonSerializationContext context) { JsonObject totals = new JsonObject(); totals.addProperty(RESOURCE_NAME_KEY, src.resourceName.get()); @@ -306,5 +419,4 @@ public class PayloadSampleReport implements JsonSerializer return totals; } } - }