mirror of https://github.com/apache/nifi.git
Adjusting handling of map to cache data items on an instance basis.
This commit is contained in:
parent
484687a67b
commit
973b493386
|
@ -40,7 +40,6 @@ import org.apache.nifi.processor.exception.ProcessException;
|
||||||
import org.apache.nifi.processor.io.OutputStreamCallback;
|
import org.apache.nifi.processor.io.OutputStreamCallback;
|
||||||
import org.apache.nifi.stream.io.BufferedOutputStream;
|
import org.apache.nifi.stream.io.BufferedOutputStream;
|
||||||
import org.apache.nifi.util.ObjectHolder;
|
import org.apache.nifi.util.ObjectHolder;
|
||||||
import org.apache.nifi.util.Tuple;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
|
@ -96,7 +95,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
|
||||||
private Set<Relationship> relationships;
|
private Set<Relationship> relationships;
|
||||||
private List<PropertyDescriptor> properties;
|
private List<PropertyDescriptor> properties;
|
||||||
|
|
||||||
private ConcurrentMap<String, Tuple<String, JsonPath>> cachedJsonPathMap = new ConcurrentHashMap<>();
|
private final ConcurrentMap<String, JsonPath> cachedJsonPathMap = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void init(final ProcessorInitializationContext context) {
|
protected void init(final ProcessorInitializationContext context) {
|
||||||
|
@ -153,13 +152,12 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
|
||||||
.addValidator(new JsonPathValidator() {
|
.addValidator(new JsonPathValidator() {
|
||||||
@Override
|
@Override
|
||||||
public void cacheComputedValue(String subject, String input, JsonPath computedJsonPath) {
|
public void cacheComputedValue(String subject, String input, JsonPath computedJsonPath) {
|
||||||
cachedJsonPathMap.put(subject, new Tuple<>(input, computedJsonPath));
|
cachedJsonPathMap.put(input, computedJsonPath);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isStale(String subject, String input) {
|
public boolean isStale(String subject, String input) {
|
||||||
return cachedJsonPathMap.get(subject) == null;
|
return cachedJsonPathMap.get(input) == null;
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.required(false)
|
.required(false)
|
||||||
|
@ -171,7 +169,9 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
|
||||||
public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
|
public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
|
||||||
if (descriptor.isDynamic()) {
|
if (descriptor.isDynamic()) {
|
||||||
if (!StringUtils.equals(oldValue, newValue)) {
|
if (!StringUtils.equals(oldValue, newValue)) {
|
||||||
cachedJsonPathMap.remove(descriptor.getName());
|
if (oldValue != null) {
|
||||||
|
cachedJsonPathMap.remove(oldValue);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -185,7 +185,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
|
||||||
public void onRemoved(ProcessContext processContext) {
|
public void onRemoved(ProcessContext processContext) {
|
||||||
for (PropertyDescriptor propertyDescriptor : getPropertyDescriptors()) {
|
for (PropertyDescriptor propertyDescriptor : getPropertyDescriptors()) {
|
||||||
if (propertyDescriptor.isDynamic()) {
|
if (propertyDescriptor.isDynamic()) {
|
||||||
cachedJsonPathMap.remove(propertyDescriptor.getName());
|
cachedJsonPathMap.remove(processContext.getProperty(propertyDescriptor).getValue());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,8 +26,9 @@ import org.apache.nifi.annotation.behavior.SideEffectFree;
|
||||||
import org.apache.nifi.annotation.behavior.SupportsBatching;
|
import org.apache.nifi.annotation.behavior.SupportsBatching;
|
||||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||||
import org.apache.nifi.annotation.documentation.Tags;
|
import org.apache.nifi.annotation.documentation.Tags;
|
||||||
import org.apache.nifi.annotation.lifecycle.OnRemoved;
|
|
||||||
import org.apache.nifi.components.PropertyDescriptor;
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.components.ValidationContext;
|
||||||
|
import org.apache.nifi.components.ValidationResult;
|
||||||
import org.apache.nifi.flowfile.FlowFile;
|
import org.apache.nifi.flowfile.FlowFile;
|
||||||
import org.apache.nifi.logging.ProcessorLog;
|
import org.apache.nifi.logging.ProcessorLog;
|
||||||
import org.apache.nifi.processor.ProcessContext;
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
|
@ -35,13 +36,13 @@ import org.apache.nifi.processor.ProcessSession;
|
||||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||||
import org.apache.nifi.processor.Relationship;
|
import org.apache.nifi.processor.Relationship;
|
||||||
import org.apache.nifi.processor.io.OutputStreamCallback;
|
import org.apache.nifi.processor.io.OutputStreamCallback;
|
||||||
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
import java.util.concurrent.ConcurrentMap;
|
|
||||||
|
|
||||||
@EventDriven
|
@EventDriven
|
||||||
@SideEffectFree
|
@SideEffectFree
|
||||||
|
@ -56,18 +57,8 @@ public class SplitJson extends AbstractJsonPathProcessor {
|
||||||
public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder()
|
public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder()
|
||||||
.name("JsonPath Expression")
|
.name("JsonPath Expression")
|
||||||
.description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
|
.description("A JsonPath expression that indicates the array element to split into JSON/scalar fragments.")
|
||||||
|
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||||
.required(true)
|
.required(true)
|
||||||
.addValidator(new JsonPathValidator() {
|
|
||||||
@Override
|
|
||||||
public void cacheComputedValue(String subject, String input, JsonPath computedJson) {
|
|
||||||
JSON_PATH_MAP.put(input, computedJson);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean isStale(String subject, String input) {
|
|
||||||
return JSON_PATH_MAP.get(input) == null;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
public static final Relationship REL_ORIGINAL = new Relationship.Builder().name("original").description("The original FlowFile that was split into segments. If the FlowFile fails processing, nothing will be sent to this relationship").build();
|
public static final Relationship REL_ORIGINAL = new Relationship.Builder().name("original").description("The original FlowFile that was split into segments. If the FlowFile fails processing, nothing will be sent to this relationship").build();
|
||||||
|
@ -77,7 +68,7 @@ public class SplitJson extends AbstractJsonPathProcessor {
|
||||||
private List<PropertyDescriptor> properties;
|
private List<PropertyDescriptor> properties;
|
||||||
private Set<Relationship> relationships;
|
private Set<Relationship> relationships;
|
||||||
|
|
||||||
private static final ConcurrentMap<String, JsonPath> JSON_PATH_MAP = new ConcurrentHashMap();
|
private final AtomicReference<JsonPath> JSON_PATH_REF = new AtomicReference<>();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void init(final ProcessorInitializationContext context) {
|
protected void init(final ProcessorInitializationContext context) {
|
||||||
|
@ -106,21 +97,30 @@ public class SplitJson extends AbstractJsonPathProcessor {
|
||||||
public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
|
public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
|
||||||
if (descriptor.equals(ARRAY_JSON_PATH_EXPRESSION)) {
|
if (descriptor.equals(ARRAY_JSON_PATH_EXPRESSION)) {
|
||||||
if (!StringUtils.equals(oldValue, newValue)) {
|
if (!StringUtils.equals(oldValue, newValue)) {
|
||||||
|
if (oldValue != null) {
|
||||||
// clear the cached item
|
// clear the cached item
|
||||||
JSON_PATH_MAP.remove(oldValue);
|
JSON_PATH_REF.set(null);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
@Override
|
||||||
* Provides cleanup of the map for any JsonPath values that may have been created. This will remove common values
|
protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
|
||||||
* shared between multiple instances, but will be regenerated when the next validation cycle occurs as a result of
|
JsonPathValidator validator = new JsonPathValidator() {
|
||||||
* isStale()
|
@Override
|
||||||
*/
|
public void cacheComputedValue(String subject, String input, JsonPath computedJson) {
|
||||||
@OnRemoved
|
JSON_PATH_REF.set(computedJson);
|
||||||
public void onRemoved(ProcessContext processContext) {
|
}
|
||||||
String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
|
|
||||||
JSON_PATH_MAP.remove(jsonPathExpression);
|
@Override
|
||||||
|
public boolean isStale(String subject, String input) {
|
||||||
|
return JSON_PATH_REF.get() == null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
String value = validationContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
|
||||||
|
return Collections.singleton(validator.validate(ARRAY_JSON_PATH_EXPRESSION.getName(), value, validationContext));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -141,8 +141,7 @@ public class SplitJson extends AbstractJsonPathProcessor {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
String jsonPathExpression = processContext.getProperty(ARRAY_JSON_PATH_EXPRESSION).getValue();
|
final JsonPath jsonPath = JSON_PATH_REF.get();
|
||||||
final JsonPath jsonPath = JSON_PATH_MAP.get(jsonPathExpression);
|
|
||||||
|
|
||||||
final List<FlowFile> segments = new ArrayList<>();
|
final List<FlowFile> segments = new ArrayList<>();
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue