mirror of https://github.com/apache/nifi.git
NIFI-11174 Removed deprecated Standard Processors
- Base64EncodeContent - GetHTTP - GetJMSQueue - GetJMSTopic - HashAttribute - HashContent - PostHTTP - PutJMS Removed deprecated Proxy properties from InvokeHTTP - Proxy Host - Proxy Port - Proxy Type - Proxy Username - Proxy Password Signed-off-by: Pierre Villard <pierre.villard.fr@gmail.com> This closes #6953.
This commit is contained in:
exceptionfactory
Pierre Villard
parent
9c5ae0d098
commit
4f11550e97
|
|
@ -1,159 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64InputStream;
|
||||
import org.apache.commons.codec.binary.Base64OutputStream;
|
||||
import org.apache.nifi.annotation.behavior.EventDriven;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.SideEffectFree;
|
||||
import org.apache.nifi.annotation.behavior.SupportsBatching;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.AbstractProcessor;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processor.io.StreamCallback;
|
||||
import org.apache.nifi.processors.standard.util.ValidatingBase64InputStream;
|
||||
import org.apache.nifi.util.StopWatch;
|
||||
|
||||
@EventDriven
|
||||
@SideEffectFree
|
||||
@SupportsBatching
|
||||
@Tags({"encode", "base64"})
|
||||
@CapabilityDescription("Encodes or decodes content to and from base64")
|
||||
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||
@DeprecationNotice(
|
||||
alternatives = EncodeContent.class,
|
||||
reason = "EncodeContent supports Base64 and additional encoding schemes"
|
||||
)
|
||||
public class Base64EncodeContent extends AbstractProcessor {
|
||||
|
||||
public static final String ENCODE_MODE = "Encode";
|
||||
public static final String DECODE_MODE = "Decode";
|
||||
|
||||
public static final PropertyDescriptor MODE = new PropertyDescriptor.Builder()
|
||||
.name("Mode")
|
||||
.description("Specifies whether the content should be encoded or decoded")
|
||||
.required(true)
|
||||
.allowableValues(ENCODE_MODE, DECODE_MODE)
|
||||
.defaultValue(ENCODE_MODE)
|
||||
.build();
|
||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||
.name("success")
|
||||
.description("Any FlowFile that is successfully encoded or decoded will be routed to success")
|
||||
.build();
|
||||
public static final Relationship REL_FAILURE = new Relationship.Builder()
|
||||
.name("failure")
|
||||
.description("Any FlowFile that cannot be encoded or decoded will be routed to failure")
|
||||
.build();
|
||||
|
||||
private List<PropertyDescriptor> properties;
|
||||
private Set<Relationship> relationships;
|
||||
|
||||
@Override
|
||||
protected void init(final ProcessorInitializationContext context) {
|
||||
final List<PropertyDescriptor> properties = new ArrayList<>();
|
||||
properties.add(MODE);
|
||||
this.properties = Collections.unmodifiableList(properties);
|
||||
|
||||
final Set<Relationship> relationships = new HashSet<>();
|
||||
relationships.add(REL_SUCCESS);
|
||||
relationships.add(REL_FAILURE);
|
||||
this.relationships = Collections.unmodifiableSet(relationships);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Relationship> getRelationships() {
|
||||
return relationships;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) {
|
||||
FlowFile flowFile = session.get();
|
||||
if (flowFile == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final ComponentLog logger = getLogger();
|
||||
|
||||
boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
|
||||
try {
|
||||
final StopWatch stopWatch = new StopWatch(true);
|
||||
if (encode) {
|
||||
flowFile = session.write(flowFile, new StreamCallback() {
|
||||
@Override
|
||||
public void process(InputStream in, OutputStream out) throws IOException {
|
||||
try (Base64OutputStream bos = new Base64OutputStream(out)) {
|
||||
int len = -1;
|
||||
byte[] buf = new byte[8192];
|
||||
while ((len = in.read(buf)) > 0) {
|
||||
bos.write(buf, 0, len);
|
||||
}
|
||||
bos.flush();
|
||||
}
|
||||
}
|
||||
});
|
||||
} else {
|
||||
flowFile = session.write(flowFile, new StreamCallback() {
|
||||
@Override
|
||||
public void process(InputStream in, OutputStream out) throws IOException {
|
||||
try (Base64InputStream bis = new Base64InputStream(new ValidatingBase64InputStream(in))) {
|
||||
int len = -1;
|
||||
byte[] buf = new byte[8192];
|
||||
while ((len = bis.read(buf)) > 0) {
|
||||
out.write(buf, 0, len);
|
||||
}
|
||||
out.flush();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.info("Successfully {} {}", new Object[] {encode ? "encoded" : "decoded", flowFile});
|
||||
session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
|
||||
session.transfer(flowFile, REL_SUCCESS);
|
||||
} catch (ProcessException e) {
|
||||
logger.error("Failed to {} {} due to {}", new Object[] {encode ? "encode" : "decode", flowFile, e});
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,586 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.client.config.CookieSpecs;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.config.Registry;
|
||||
import org.apache.http.config.RegistryBuilder;
|
||||
import org.apache.http.conn.HttpClientConnectionManager;
|
||||
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
||||
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
|
||||
import org.apache.nifi.annotation.behavior.DynamicProperties;
|
||||
import org.apache.nifi.annotation.behavior.DynamicProperty;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.Stateful;
|
||||
import org.apache.nifi.annotation.behavior.WritesAttribute;
|
||||
import org.apache.nifi.annotation.behavior.WritesAttributes;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.annotation.lifecycle.OnScheduled;
|
||||
import org.apache.nifi.components.AllowableValue;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.components.PropertyValue;
|
||||
import org.apache.nifi.components.ValidationContext;
|
||||
import org.apache.nifi.components.ValidationResult;
|
||||
import org.apache.nifi.components.Validator;
|
||||
import org.apache.nifi.components.state.Scope;
|
||||
import org.apache.nifi.components.state.StateMap;
|
||||
import org.apache.nifi.expression.AttributeExpression;
|
||||
import org.apache.nifi.expression.ExpressionLanguageScope;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
import org.apache.nifi.flowfile.attributes.CoreAttributes;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.ProcessSessionFactory;
|
||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
import org.apache.nifi.processors.standard.util.HTTPUtils;
|
||||
import org.apache.nifi.ssl.SSLContextService;
|
||||
import org.apache.nifi.util.StopWatch;
|
||||
import org.apache.nifi.util.Tuple;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.apache.nifi.processors.standard.util.HTTPUtils.PROXY_HOST;
|
||||
import static org.apache.nifi.processors.standard.util.HTTPUtils.PROXY_PORT;
|
||||
|
||||
@Deprecated
|
||||
@DeprecationNotice(alternatives = {InvokeHTTP.class}, reason = "This processor is deprecated and may be removed in future releases.")
|
||||
@Tags({"get", "fetch", "poll", "http", "https", "ingest", "source", "input"})
|
||||
@InputRequirement(Requirement.INPUT_FORBIDDEN)
|
||||
@CapabilityDescription("Please be aware this processor is deprecated and may be removed in the near future. Use InvokeHTTP instead. "
|
||||
+ "Fetches data from an HTTP or HTTPS URL and writes the data to the content of a FlowFile. Once the content has been fetched, the ETag and Last Modified "
|
||||
+ "dates are remembered (if the web server supports these concepts). This allows the Processor to fetch new data only if the remote data has changed or until the state is cleared. That is, "
|
||||
+ "once the content has been fetched from the given URL, it will not be fetched again until the content on the remote server changes. Note that due to limitations on state "
|
||||
+ "management, stored \"last modified\" and etag fields never expire. If the URL in GetHttp uses Expression Language that is unbounded, there "
|
||||
+ "is the potential for Out of Memory Errors to occur.")
|
||||
@DynamicProperties({
|
||||
@DynamicProperty(name = "Header Name", value = "The Expression Language to be used to populate the header value",
|
||||
expressionLanguageScope = ExpressionLanguageScope.VARIABLE_REGISTRY,
|
||||
description = "The additional headers to be sent by the processor whenever making a new HTTP request. \n " +
|
||||
"Setting a dynamic property name to XYZ and value to ${attribute} will result in the header 'XYZ: attribute_value' being sent to the HTTP endpoint"),
|
||||
})
|
||||
@WritesAttributes({
|
||||
@WritesAttribute(attribute = "filename", description = "The filename is set to the name of the file on the remote server"),
|
||||
@WritesAttribute(attribute = "mime.type", description = "The MIME Type of the FlowFile, as reported by the HTTP Content-Type header")
|
||||
})
|
||||
@Stateful(scopes = {Scope.LOCAL}, description = "Stores Last Modified Time and ETag headers returned by server so that the same data will not be fetched multiple times.")
|
||||
public class GetHTTP extends AbstractSessionFactoryProcessor {
|
||||
|
||||
static final int PERSISTENCE_INTERVAL_MSEC = 10000;
|
||||
|
||||
public static final String HEADER_IF_NONE_MATCH = "If-None-Match";
|
||||
public static final String HEADER_IF_MODIFIED_SINCE = "If-Modified-Since";
|
||||
public static final String HEADER_ACCEPT = "Accept";
|
||||
public static final String HEADER_LAST_MODIFIED = "Last-Modified";
|
||||
public static final String HEADER_ETAG = "ETag";
|
||||
public static final int NOT_MODIFIED = 304;
|
||||
|
||||
public static final PropertyDescriptor URL = new PropertyDescriptor.Builder()
|
||||
.name("URL")
|
||||
.description("The URL to pull from")
|
||||
.required(true)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.addValidator(StandardValidators.URL_VALIDATOR)
|
||||
.addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("https?\\://.*")))
|
||||
.build();
|
||||
public static final PropertyDescriptor FOLLOW_REDIRECTS = new PropertyDescriptor.Builder()
|
||||
.name("Follow Redirects")
|
||||
.description("If we receive a 3xx HTTP Status Code from the server, indicates whether or not we should follow the redirect that the server specifies")
|
||||
.defaultValue("false")
|
||||
.allowableValues("true", "false")
|
||||
.build();
|
||||
public static final PropertyDescriptor CONNECTION_TIMEOUT = new PropertyDescriptor.Builder()
|
||||
.name("Connection Timeout")
|
||||
.description("How long to wait when attempting to connect to the remote server before giving up")
|
||||
.required(true)
|
||||
.defaultValue("30 sec")
|
||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor ACCEPT_CONTENT_TYPE = new PropertyDescriptor.Builder()
|
||||
.name("Accept Content-Type")
|
||||
.description("If specified, requests will only accept the provided Content-Type")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor DATA_TIMEOUT = new PropertyDescriptor.Builder()
|
||||
.name("Data Timeout")
|
||||
.description("How long to wait between receiving segments of data from the remote server before giving up and discarding the partial file")
|
||||
.required(true)
|
||||
.defaultValue("30 sec")
|
||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor FILENAME = new PropertyDescriptor.Builder()
|
||||
.name("Filename")
|
||||
.description("The filename to assign to the file when pulled")
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING))
|
||||
.required(true)
|
||||
.build();
|
||||
public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder()
|
||||
.name("Username")
|
||||
.description("Username required to access the URL")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder()
|
||||
.name("Password")
|
||||
.description("Password required to access the URL")
|
||||
.required(false)
|
||||
.sensitive(true)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor USER_AGENT = new PropertyDescriptor.Builder()
|
||||
.name("User Agent")
|
||||
.description("What to report as the User Agent when we connect to the remote server")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||
.name("SSL Context Service")
|
||||
.description("The Controller Service to use in order to obtain an SSL Context")
|
||||
.required(false)
|
||||
.identifiesControllerService(SSLContextService.class)
|
||||
.build();
|
||||
|
||||
public static final String DEFAULT_COOKIE_POLICY_STR = "default";
|
||||
public static final String STANDARD_COOKIE_POLICY_STR = "standard";
|
||||
public static final String STRICT_COOKIE_POLICY_STR = "strict";
|
||||
public static final String NETSCAPE_COOKIE_POLICY_STR = "netscape";
|
||||
public static final String IGNORE_COOKIE_POLICY_STR = "ignore";
|
||||
public static final AllowableValue DEFAULT_COOKIE_POLICY = new AllowableValue(DEFAULT_COOKIE_POLICY_STR, DEFAULT_COOKIE_POLICY_STR,
|
||||
"Default cookie policy that provides a higher degree of compatibility with common cookie management of popular HTTP agents for non-standard (Netscape style) cookies.");
|
||||
public static final AllowableValue STANDARD_COOKIE_POLICY = new AllowableValue(STANDARD_COOKIE_POLICY_STR, STANDARD_COOKIE_POLICY_STR,
|
||||
"RFC 6265 compliant cookie policy (interoperability profile).");
|
||||
public static final AllowableValue STRICT_COOKIE_POLICY = new AllowableValue(STRICT_COOKIE_POLICY_STR, STRICT_COOKIE_POLICY_STR,
|
||||
"RFC 6265 compliant cookie policy (strict profile).");
|
||||
public static final AllowableValue NETSCAPE_COOKIE_POLICY = new AllowableValue(NETSCAPE_COOKIE_POLICY_STR, NETSCAPE_COOKIE_POLICY_STR,
|
||||
"Netscape draft compliant cookie policy.");
|
||||
public static final AllowableValue IGNORE_COOKIE_POLICY = new AllowableValue(IGNORE_COOKIE_POLICY_STR, IGNORE_COOKIE_POLICY_STR,
|
||||
"A cookie policy that ignores cookies.");
|
||||
|
||||
public static final PropertyDescriptor REDIRECT_COOKIE_POLICY = new PropertyDescriptor.Builder()
|
||||
.name("redirect-cookie-policy")
|
||||
.displayName("Redirect Cookie Policy")
|
||||
.description("When a HTTP server responds to a request with a redirect, this is the cookie policy used to copy cookies to the following request.")
|
||||
.allowableValues(DEFAULT_COOKIE_POLICY, STANDARD_COOKIE_POLICY, STRICT_COOKIE_POLICY, NETSCAPE_COOKIE_POLICY, IGNORE_COOKIE_POLICY)
|
||||
.defaultValue(DEFAULT_COOKIE_POLICY_STR)
|
||||
.build();
|
||||
|
||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||
.name("success")
|
||||
.description("All files are transferred to the success relationship")
|
||||
.build();
|
||||
|
||||
public static final String LAST_MODIFIED_DATE_PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss zzz";
|
||||
|
||||
// package access to enable unit testing
|
||||
static final String ETAG = "ETag";
|
||||
static final String LAST_MODIFIED = "LastModified";
|
||||
|
||||
|
||||
private Set<Relationship> relationships;
|
||||
private List<PropertyDescriptor> properties;
|
||||
private final List<PropertyDescriptor> customHeaders = new ArrayList<>();
|
||||
|
||||
private final AtomicBoolean clearState = new AtomicBoolean(false);
|
||||
|
||||
@Override
|
||||
protected void init(final ProcessorInitializationContext context) {
|
||||
final Set<Relationship> relationships = new HashSet<>();
|
||||
relationships.add(REL_SUCCESS);
|
||||
this.relationships = Collections.unmodifiableSet(relationships);
|
||||
|
||||
final List<PropertyDescriptor> properties = new ArrayList<>();
|
||||
properties.add(URL);
|
||||
properties.add(FILENAME);
|
||||
properties.add(SSL_CONTEXT_SERVICE);
|
||||
properties.add(USERNAME);
|
||||
properties.add(PASSWORD);
|
||||
properties.add(CONNECTION_TIMEOUT);
|
||||
properties.add(DATA_TIMEOUT);
|
||||
properties.add(USER_AGENT);
|
||||
properties.add(ACCEPT_CONTENT_TYPE);
|
||||
properties.add(FOLLOW_REDIRECTS);
|
||||
properties.add(REDIRECT_COOKIE_POLICY);
|
||||
properties.add(HTTPUtils.PROXY_CONFIGURATION_SERVICE);
|
||||
properties.add(PROXY_HOST);
|
||||
properties.add(PROXY_PORT);
|
||||
this.properties = Collections.unmodifiableList(properties);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Relationship> getRelationships() {
|
||||
return relationships;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
|
||||
clearState.set(true);
|
||||
}
|
||||
|
||||
@OnScheduled
|
||||
public void onScheduled(final ProcessContext context) throws IOException {
|
||||
if (clearState.getAndSet(false)) {
|
||||
context.getStateManager().clear(Scope.LOCAL);
|
||||
}
|
||||
if (customHeaders.size() == 0) {
|
||||
for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
|
||||
// only add the custom defined Headers (i.e. dynamic properties)
|
||||
if (!getSupportedPropertyDescriptors().contains(property.getKey())) {
|
||||
customHeaders.add(property.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<ValidationResult> customValidate(final ValidationContext context) {
|
||||
final Collection<ValidationResult> results = new ArrayList<>();
|
||||
|
||||
if (context.getProperty(URL).evaluateAttributeExpressions().getValue().startsWith("https") && context.getProperty(SSL_CONTEXT_SERVICE).getValue() == null) {
|
||||
results.add(new ValidationResult.Builder()
|
||||
.explanation("URL is set to HTTPS protocol but no SSLContext has been specified")
|
||||
.valid(false)
|
||||
.subject("SSL Context")
|
||||
.build());
|
||||
}
|
||||
|
||||
HTTPUtils.validateProxyProperties(context, results);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
|
||||
return new PropertyDescriptor.Builder()
|
||||
.name(propertyDescriptorName)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.addValidator(Validator.VALID)
|
||||
.required(false)
|
||||
.dynamic(true)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
|
||||
final ComponentLog logger = getLogger();
|
||||
|
||||
final ProcessSession session = sessionFactory.createSession();
|
||||
final FlowFile incomingFlowFile = session.get();
|
||||
if (incomingFlowFile != null) {
|
||||
session.transfer(incomingFlowFile, REL_SUCCESS);
|
||||
logger.warn("found FlowFile {} in input queue; transferring to success", new Object[]{incomingFlowFile});
|
||||
}
|
||||
|
||||
// get the URL
|
||||
final String url = context.getProperty(URL).evaluateAttributeExpressions().getValue();
|
||||
final URI uri;
|
||||
String source = url;
|
||||
try {
|
||||
uri = new URI(url);
|
||||
source = uri.getHost();
|
||||
} catch (final URISyntaxException swallow) {
|
||||
// this won't happen as the url has already been validated
|
||||
}
|
||||
|
||||
// get the ssl context service
|
||||
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
|
||||
|
||||
// create the connection manager
|
||||
final HttpClientConnectionManager conMan;
|
||||
if (sslContextService == null) {
|
||||
conMan = new BasicHttpClientConnectionManager();
|
||||
} else {
|
||||
final SSLContext sslContext;
|
||||
try {
|
||||
sslContext = sslContextService.createContext();
|
||||
} catch (final Exception e) {
|
||||
throw new ProcessException(e);
|
||||
}
|
||||
|
||||
final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
|
||||
|
||||
// Also include a plain socket factory for regular http connections (especially proxies)
|
||||
final Registry<ConnectionSocketFactory> socketFactoryRegistry =
|
||||
RegistryBuilder.<ConnectionSocketFactory>create()
|
||||
.register("https", sslsf)
|
||||
.register("http", PlainConnectionSocketFactory.getSocketFactory())
|
||||
.build();
|
||||
|
||||
conMan = new BasicHttpClientConnectionManager(socketFactoryRegistry);
|
||||
}
|
||||
|
||||
try {
|
||||
// build the request configuration
|
||||
final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
|
||||
requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean());
|
||||
switch (context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) {
|
||||
case STANDARD_COOKIE_POLICY_STR:
|
||||
requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
|
||||
break;
|
||||
case STRICT_COOKIE_POLICY_STR:
|
||||
requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT);
|
||||
break;
|
||||
case NETSCAPE_COOKIE_POLICY_STR:
|
||||
requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE);
|
||||
break;
|
||||
case IGNORE_COOKIE_POLICY_STR:
|
||||
requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
|
||||
break;
|
||||
case DEFAULT_COOKIE_POLICY_STR:
|
||||
default:
|
||||
requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);
|
||||
}
|
||||
|
||||
// build the http client
|
||||
final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
|
||||
clientBuilder.setConnectionManager(conMan);
|
||||
|
||||
// include the user agent
|
||||
final String userAgent = context.getProperty(USER_AGENT).getValue();
|
||||
if (userAgent != null) {
|
||||
clientBuilder.setUserAgent(userAgent);
|
||||
}
|
||||
|
||||
// set the ssl context if necessary
|
||||
if (sslContextService != null) {
|
||||
final SSLContext sslContext = sslContextService.createContext();
|
||||
clientBuilder.setSSLContext(sslContext);
|
||||
}
|
||||
|
||||
final String username = context.getProperty(USERNAME).getValue();
|
||||
final String password = context.getProperty(PASSWORD).getValue();
|
||||
|
||||
// set the credentials if appropriate
|
||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
if (username != null) {
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
|
||||
}
|
||||
|
||||
// Set the proxy if specified
|
||||
HTTPUtils.setProxy(context, clientBuilder, credentialsProvider);
|
||||
|
||||
// create request
|
||||
final HttpGet get = new HttpGet(url);
|
||||
get.setConfig(requestConfigBuilder.build());
|
||||
|
||||
final StateMap beforeStateMap;
|
||||
|
||||
try {
|
||||
beforeStateMap = session.getState(Scope.LOCAL);
|
||||
final String lastModified = beforeStateMap.get(LAST_MODIFIED + ":" + url);
|
||||
if (lastModified != null) {
|
||||
get.addHeader(HEADER_IF_MODIFIED_SINCE, parseStateValue(lastModified).getValue());
|
||||
}
|
||||
|
||||
final String etag = beforeStateMap.get(ETAG + ":" + url);
|
||||
if (etag != null) {
|
||||
get.addHeader(HEADER_IF_NONE_MATCH, parseStateValue(etag).getValue());
|
||||
}
|
||||
} catch (final IOException ioe) {
|
||||
throw new ProcessException(ioe);
|
||||
}
|
||||
|
||||
final String accept = context.getProperty(ACCEPT_CONTENT_TYPE).getValue();
|
||||
if (accept != null) {
|
||||
get.addHeader(HEADER_ACCEPT, accept);
|
||||
}
|
||||
|
||||
// Add dynamic headers
|
||||
|
||||
PropertyValue customHeaderValue;
|
||||
for (PropertyDescriptor customProperty : customHeaders) {
|
||||
customHeaderValue = context.getProperty(customProperty).evaluateAttributeExpressions();
|
||||
if (StringUtils.isNotBlank(customHeaderValue.getValue())) {
|
||||
get.addHeader(customProperty.getName(), customHeaderValue.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// create the http client
|
||||
try (final CloseableHttpClient client = clientBuilder.build()) {
|
||||
// NOTE: including this inner try in order to swallow exceptions on close
|
||||
try {
|
||||
final StopWatch stopWatch = new StopWatch(true);
|
||||
final HttpResponse response = client.execute(get);
|
||||
final int statusCode = response.getStatusLine().getStatusCode();
|
||||
if (statusCode == NOT_MODIFIED) {
|
||||
logger.info("content not retrieved because server returned HTTP Status Code {}: Not Modified", new Object[]{NOT_MODIFIED});
|
||||
context.yield();
|
||||
// doing a commit in case there were flow files in the input queue
|
||||
session.commitAsync();
|
||||
return;
|
||||
}
|
||||
final String statusExplanation = response.getStatusLine().getReasonPhrase();
|
||||
|
||||
if ((statusCode >= 300) || (statusCode == 204)) {
|
||||
logger.error("received status code {}:{} from {}", new Object[]{statusCode, statusExplanation, url});
|
||||
// doing a commit in case there were flow files in the input queue
|
||||
session.commitAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
FlowFile flowFile = session.create();
|
||||
flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
|
||||
flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", source);
|
||||
flowFile = session.importFrom(response.getEntity().getContent(), flowFile);
|
||||
|
||||
final Header contentTypeHeader = response.getFirstHeader("Content-Type");
|
||||
if (contentTypeHeader != null) {
|
||||
final String contentType = contentTypeHeader.getValue();
|
||||
if (!contentType.trim().isEmpty()) {
|
||||
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), contentType.trim());
|
||||
}
|
||||
}
|
||||
|
||||
final long flowFileSize = flowFile.getSize();
|
||||
stopWatch.stop();
|
||||
final String dataRate = stopWatch.calculateDataRate(flowFileSize);
|
||||
session.getProvenanceReporter().receive(flowFile, url, stopWatch.getDuration(TimeUnit.MILLISECONDS));
|
||||
session.transfer(flowFile, REL_SUCCESS);
|
||||
logger.info("Successfully received {} from {} at a rate of {}; transferred to success", new Object[]{flowFile, url, dataRate});
|
||||
|
||||
updateStateMap(context, session, response, beforeStateMap, url);
|
||||
session.commitAsync();
|
||||
|
||||
} catch (final IOException e) {
|
||||
context.yield();
|
||||
session.rollback();
|
||||
logger.error("Failed to retrieve file from {} due to {}; rolling back session", new Object[]{url, e.getMessage()}, e);
|
||||
throw new ProcessException(e);
|
||||
} catch (final Throwable t) {
|
||||
context.yield();
|
||||
session.rollback();
|
||||
logger.error("Failed to process due to {}; rolling back session", new Object[]{t.getMessage()}, t);
|
||||
throw t;
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
logger.debug("Error closing client due to {}, continuing.", new Object[]{e.getMessage()});
|
||||
}
|
||||
} finally {
|
||||
conMan.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
private void updateStateMap(final ProcessContext context, final ProcessSession session, HttpResponse response, StateMap beforeStateMap, String url) {
|
||||
try {
|
||||
Map<String, String> workingMap = new HashMap<>();
|
||||
workingMap.putAll(beforeStateMap.toMap());
|
||||
StateMap oldValue = beforeStateMap;
|
||||
|
||||
long currentTime = System.currentTimeMillis();
|
||||
|
||||
final Header receivedLastModified = response.getFirstHeader(HEADER_LAST_MODIFIED);
|
||||
if (receivedLastModified != null) {
|
||||
workingMap.put(LAST_MODIFIED + ":" + url, currentTime + ":" + receivedLastModified.getValue());
|
||||
}
|
||||
|
||||
final Header receivedEtag = response.getFirstHeader(HEADER_ETAG);
|
||||
if (receivedEtag != null) {
|
||||
workingMap.put(ETAG + ":" + url, currentTime + ":" + receivedEtag.getValue());
|
||||
}
|
||||
|
||||
boolean replaceSucceeded = session.replaceState(oldValue, workingMap, Scope.LOCAL);
|
||||
boolean changed;
|
||||
|
||||
while (!replaceSucceeded) {
|
||||
oldValue = session.getState(Scope.LOCAL);
|
||||
workingMap.clear();
|
||||
workingMap.putAll(oldValue.toMap());
|
||||
|
||||
changed = false;
|
||||
|
||||
if (receivedLastModified != null) {
|
||||
Tuple<String, String> storedLastModifiedTuple = parseStateValue(workingMap.get(LAST_MODIFIED + ":" + url));
|
||||
|
||||
if (Long.parseLong(storedLastModifiedTuple.getKey()) < currentTime) {
|
||||
workingMap.put(LAST_MODIFIED + ":" + url, currentTime + ":" + receivedLastModified.getValue());
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (receivedEtag != null) {
|
||||
Tuple<String, String> storedLastModifiedTuple = parseStateValue(workingMap.get(ETAG + ":" + url));
|
||||
|
||||
if (Long.parseLong(storedLastModifiedTuple.getKey()) < currentTime) {
|
||||
workingMap.put(ETAG + ":" + url, currentTime + ":" + receivedEtag.getValue());
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
replaceSucceeded = session.replaceState(oldValue, workingMap, Scope.LOCAL);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (final IOException ioe) {
|
||||
throw new ProcessException(ioe);
|
||||
}
|
||||
}
|
||||
|
||||
protected static Tuple<String, String> parseStateValue(String mapValue) {
|
||||
int indexOfColon = mapValue.indexOf(":");
|
||||
|
||||
String timestamp = mapValue.substring(0, indexOfColon);
|
||||
String value = mapValue.substring(indexOfColon + 1);
|
||||
return new Tuple<>(timestamp, value);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import javax.jms.JMSException;
|
||||
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.SeeAlso;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processors.standard.util.JmsFactory;
|
||||
import org.apache.nifi.processors.standard.util.WrappedMessageConsumer;
|
||||
|
||||
@Deprecated
|
||||
@DeprecationNotice(classNames = {"org.apache.nifi.jms.processors.ConsumeJMS"}, reason = "This processor is deprecated and may be removed in future releases. ")
|
||||
@TriggerWhenEmpty
|
||||
@InputRequirement(Requirement.INPUT_FORBIDDEN)
|
||||
@Tags({"jms", "queue", "listen", "get", "pull", "source", "consume", "consumer"})
|
||||
@CapabilityDescription("Pulls messages from a ActiveMQ JMS Queue, creating a FlowFile for each JMS Message or bundle of messages, as configured")
|
||||
@SeeAlso({PutJMS.class})
|
||||
public class GetJMSQueue extends JmsConsumer {
|
||||
|
||||
private final Queue<WrappedMessageConsumer> consumerQueue = new LinkedBlockingQueue<>();
|
||||
|
||||
@OnStopped
|
||||
public void cleanupResources() {
|
||||
WrappedMessageConsumer wrappedConsumer = consumerQueue.poll();
|
||||
while (wrappedConsumer != null) {
|
||||
wrappedConsumer.close(getLogger());
|
||||
wrappedConsumer = consumerQueue.poll();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
|
||||
final ComponentLog logger = getLogger();
|
||||
|
||||
WrappedMessageConsumer wrappedConsumer = consumerQueue.poll();
|
||||
if (wrappedConsumer == null) {
|
||||
try {
|
||||
wrappedConsumer = JmsFactory.createQueueMessageConsumer(context);
|
||||
} catch (JMSException e) {
|
||||
logger.error("Failed to connect to JMS Server due to {}", e);
|
||||
context.yield();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
super.consume(context, session, wrappedConsumer);
|
||||
} finally {
|
||||
if (!wrappedConsumer.isClosed()) {
|
||||
consumerQueue.offer(wrappedConsumer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,356 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.CLIENT_ID_PREFIX;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.DURABLE_SUBSCRIPTION;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.JMS_PROVIDER;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.PASSWORD;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.URL;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.USERNAME;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.jms.Connection;
|
||||
import javax.jms.InvalidDestinationException;
|
||||
import javax.jms.JMSException;
|
||||
import javax.jms.Session;
|
||||
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.PrimaryNodeOnly;
|
||||
import org.apache.nifi.annotation.behavior.TriggerSerially;
|
||||
import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.SeeAlso;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.annotation.lifecycle.OnRemoved;
|
||||
import org.apache.nifi.annotation.lifecycle.OnScheduled;
|
||||
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processors.standard.util.JmsFactory;
|
||||
import org.apache.nifi.processors.standard.util.JmsProperties;
|
||||
import org.apache.nifi.processors.standard.util.WrappedMessageConsumer;
|
||||
|
||||
@PrimaryNodeOnly
|
||||
@Deprecated
|
||||
@DeprecationNotice(classNames = {"org.apache.nifi.jms.processors.ConsumeJMS"}, reason = "This processor is deprecated and may be removed in future releases.")
|
||||
@TriggerSerially
|
||||
@TriggerWhenEmpty
|
||||
@InputRequirement(Requirement.INPUT_FORBIDDEN)
|
||||
@Tags({"jms", "topic", "subscription", "durable", "non-durable", "listen", "get", "pull", "source", "consume", "consumer"})
|
||||
@CapabilityDescription("Pulls messages from a ActiveMQ JMS Topic, creating a FlowFile for each JMS Message or bundle of messages, as configured.")
|
||||
@SeeAlso({PutJMS.class })
|
||||
public class GetJMSTopic extends JmsConsumer {
|
||||
|
||||
public static final String SUBSCRIPTION_NAME_PROPERTY = "subscription.name";
|
||||
private volatile WrappedMessageConsumer wrappedConsumer = null;
|
||||
|
||||
private final List<PropertyDescriptor> properties;
|
||||
|
||||
public GetJMSTopic() {
|
||||
super();
|
||||
|
||||
final List<PropertyDescriptor> props = new ArrayList<>(super.getSupportedPropertyDescriptors());
|
||||
props.add(JmsProperties.DURABLE_SUBSCRIPTION);
|
||||
properties = Collections.unmodifiableList(props);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@OnStopped
|
||||
public void cleanupResources() {
|
||||
final WrappedMessageConsumer consumer = this.wrappedConsumer;
|
||||
if (consumer != null) {
|
||||
try {
|
||||
consumer.close(getLogger());
|
||||
} finally {
|
||||
this.wrappedConsumer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Path getSubscriptionPath() {
|
||||
return Paths.get("conf").resolve("jms-subscription-" + getIdentifier());
|
||||
}
|
||||
|
||||
@OnScheduled
|
||||
public void handleSubscriptions(final ProcessContext context) throws IOException, JMSException {
|
||||
boolean usingDurableSubscription = context.getProperty(DURABLE_SUBSCRIPTION).asBoolean();
|
||||
final Properties persistedProps = getSubscriptionPropertiesFromFile();
|
||||
final Properties currentProps = getSubscriptionPropertiesFromContext(context);
|
||||
if (persistedProps == null) {
|
||||
if (usingDurableSubscription) {
|
||||
persistSubscriptionInfo(context); // properties have not yet been persisted.
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// decrypt the passwords so the persisted and current properties can be compared...
|
||||
// we can modify this properties instance since the unsubscribe method will reload
|
||||
// the properties from disk
|
||||
decryptPassword(persistedProps, context);
|
||||
decryptPassword(currentProps, context);
|
||||
|
||||
// check if current values are the same as the persisted values.
|
||||
boolean same = true;
|
||||
for (final Map.Entry<Object, Object> entry : persistedProps.entrySet()) {
|
||||
final Object key = entry.getKey();
|
||||
|
||||
final Object value = entry.getValue();
|
||||
final Object curVal = currentProps.get(key);
|
||||
if (value == null && curVal == null) {
|
||||
continue;
|
||||
}
|
||||
if (value == null || curVal == null) {
|
||||
same = false;
|
||||
break;
|
||||
}
|
||||
if (SUBSCRIPTION_NAME_PROPERTY.equals(key)) {
|
||||
// ignore the random UUID part of the subscription name
|
||||
if (!JmsFactory.clientIdPrefixEquals(value.toString(), curVal.toString())) {
|
||||
same = false;
|
||||
break;
|
||||
}
|
||||
} else if (!value.equals(curVal)) {
|
||||
same = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (same && usingDurableSubscription) {
|
||||
return; // properties are the same.
|
||||
}
|
||||
|
||||
// unsubscribe from the old subscription.
|
||||
try {
|
||||
unsubscribe(context);
|
||||
} catch (final InvalidDestinationException e) {
|
||||
getLogger().warn("Failed to unsubscribe from subscription due to {}; subscription does not appear to be active, so ignoring it", new Object[]{e});
|
||||
}
|
||||
|
||||
// we've now got a new subscription, so we must persist that new info before we create the subscription.
|
||||
if (usingDurableSubscription) {
|
||||
persistSubscriptionInfo(context);
|
||||
} else {
|
||||
// remove old subscription info if it was persisted
|
||||
try {
|
||||
Files.delete(getSubscriptionPath());
|
||||
} catch (Exception ignore) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to locate the password in the specified properties. If found, decrypts it using the specified context.
|
||||
*
|
||||
* @param properties properties
|
||||
* @param context context
|
||||
*/
|
||||
protected void decryptPassword(final Properties properties, final ProcessContext context) {
|
||||
final String encryptedPassword = properties.getProperty(PASSWORD.getName());
|
||||
|
||||
// if the is in the properties, decrypt it
|
||||
if (encryptedPassword != null) {
|
||||
properties.put(PASSWORD.getName(), context.decrypt(encryptedPassword));
|
||||
}
|
||||
}
|
||||
|
||||
@OnRemoved
|
||||
public void onRemoved(final ProcessContext context) throws IOException, JMSException {
|
||||
// unsubscribe from the old subscription.
|
||||
unsubscribe(context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Persists the subscription details for future use.
|
||||
*
|
||||
* @param context context
|
||||
* @throws IOException ex
|
||||
*/
|
||||
private void persistSubscriptionInfo(final ProcessContext context) throws IOException {
|
||||
final Properties props = getSubscriptionPropertiesFromContext(context);
|
||||
try (final OutputStream out = Files.newOutputStream(getSubscriptionPath())) {
|
||||
props.store(out, null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the subscription details from the specified context. Note: if a password is set, the resulting entry will be encrypted.
|
||||
*
|
||||
* @param context context
|
||||
* @return Returns the subscription details from the specified context
|
||||
*/
|
||||
private Properties getSubscriptionPropertiesFromContext(final ProcessContext context) {
|
||||
final String unencryptedPassword = context.getProperty(PASSWORD).getValue();
|
||||
final String encryptedPassword = (unencryptedPassword == null) ? null : context.encrypt(unencryptedPassword);
|
||||
|
||||
final Properties props = new Properties();
|
||||
props.setProperty(URL.getName(), context.getProperty(URL).getValue());
|
||||
|
||||
if (context.getProperty(USERNAME).isSet()) {
|
||||
props.setProperty(USERNAME.getName(), context.getProperty(USERNAME).getValue());
|
||||
}
|
||||
|
||||
if (encryptedPassword != null) {
|
||||
props.setProperty(PASSWORD.getName(), encryptedPassword);
|
||||
}
|
||||
|
||||
props.setProperty(SUBSCRIPTION_NAME_PROPERTY, JmsFactory.createClientId(context));
|
||||
props.setProperty(JMS_PROVIDER.getName(), context.getProperty(JMS_PROVIDER).getValue());
|
||||
|
||||
if (context.getProperty(CLIENT_ID_PREFIX).isSet()) {
|
||||
props.setProperty(CLIENT_ID_PREFIX.getName(), context.getProperty(CLIENT_ID_PREFIX).getValue());
|
||||
}
|
||||
|
||||
return props;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the subscription details from disk. Since the details are coming from disk, if a password is set, the resulting entry will be encrypted.
|
||||
*
|
||||
* @return properties
|
||||
* @throws IOException ex
|
||||
*/
|
||||
private Properties getSubscriptionPropertiesFromFile() throws IOException {
|
||||
final Path subscriptionPath = getSubscriptionPath();
|
||||
final boolean exists = Files.exists(subscriptionPath);
|
||||
if (!exists) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final Properties props = new Properties();
|
||||
try (final InputStream in = Files.newInputStream(subscriptionPath)) {
|
||||
props.load(in);
|
||||
}
|
||||
|
||||
return props;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads subscription info from the Subscription File and unsubscribes from the subscription, if the file exists; otherwise, does nothing
|
||||
*
|
||||
* @throws IOException ex
|
||||
* @throws JMSException ex
|
||||
*/
|
||||
private void unsubscribe(final ProcessContext context) throws IOException, JMSException {
|
||||
final Properties props = getSubscriptionPropertiesFromFile();
|
||||
if (props == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final String serverUrl = props.getProperty(URL.getName());
|
||||
final String username = props.getProperty(USERNAME.getName());
|
||||
final String encryptedPassword = props.getProperty(PASSWORD.getName());
|
||||
final String subscriptionName = props.getProperty(SUBSCRIPTION_NAME_PROPERTY);
|
||||
final String jmsProvider = props.getProperty(JMS_PROVIDER.getName());
|
||||
|
||||
final String password = encryptedPassword == null ? null : context.decrypt(encryptedPassword);
|
||||
|
||||
final int timeoutMillis = context.getProperty(JmsProperties.TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
|
||||
unsubscribe(serverUrl, username, password, subscriptionName, jmsProvider, timeoutMillis);
|
||||
}
|
||||
|
||||
private void unsubscribe(final String url, final String username, final String password, final String subscriptionId, final String jmsProvider, final int timeoutMillis) throws JMSException {
|
||||
final Connection connection;
|
||||
if (username == null && password == null) {
|
||||
connection = JmsFactory.createConnectionFactory(url, timeoutMillis, jmsProvider).createConnection();
|
||||
} else {
|
||||
connection = JmsFactory.createConnectionFactory(url, timeoutMillis, jmsProvider).createConnection(username, password);
|
||||
}
|
||||
|
||||
Session session = null;
|
||||
try {
|
||||
connection.setClientID(subscriptionId);
|
||||
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
|
||||
session.unsubscribe(subscriptionId);
|
||||
|
||||
getLogger().info("Successfully unsubscribed from {}, Subscription Identifier {}", new Object[]{url, subscriptionId});
|
||||
} finally {
|
||||
if (session != null) {
|
||||
try {
|
||||
session.close();
|
||||
} catch (final Exception e1) {
|
||||
getLogger().warn("Unable to close session with JMS Server due to {}; resources may not be cleaned up appropriately", new Object[]{e1});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
connection.close();
|
||||
} catch (final Exception e1) {
|
||||
getLogger().warn("Unable to close connection to JMS Server due to {}; resources may not be cleaned up appropriately", new Object[]{e1});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OnStopped
|
||||
public void onStopped() {
|
||||
final WrappedMessageConsumer consumer = this.wrappedConsumer;
|
||||
if (consumer != null) {
|
||||
consumer.close(getLogger());
|
||||
this.wrappedConsumer = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
|
||||
final ComponentLog logger = getLogger();
|
||||
|
||||
WrappedMessageConsumer consumer = this.wrappedConsumer;
|
||||
if (consumer == null || consumer.isClosed()) {
|
||||
try {
|
||||
Properties props = null;
|
||||
try {
|
||||
props = getSubscriptionPropertiesFromFile();
|
||||
} catch (IOException ignore) {
|
||||
}
|
||||
if (props == null) {
|
||||
props = getSubscriptionPropertiesFromContext(context);
|
||||
}
|
||||
String subscriptionName = props.getProperty(SUBSCRIPTION_NAME_PROPERTY);
|
||||
consumer = JmsFactory.createTopicMessageConsumer(context, subscriptionName);
|
||||
this.wrappedConsumer = consumer;
|
||||
} catch (final JMSException e) {
|
||||
logger.error("Failed to connect to JMS Server due to {}", new Object[]{e});
|
||||
context.yield();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
super.consume(context, session, consumer);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,259 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.nifi.annotation.behavior.DefaultRunDuration;
|
||||
import org.apache.nifi.annotation.behavior.DynamicProperty;
|
||||
import org.apache.nifi.annotation.behavior.EventDriven;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.SideEffectFree;
|
||||
import org.apache.nifi.annotation.behavior.SupportsBatching;
|
||||
import org.apache.nifi.annotation.behavior.WritesAttribute;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.AbstractProcessor;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
|
||||
/**
|
||||
* This processor <strong>does not calculate a cryptographic hash of one or more attributes</strong>.
|
||||
* For that behavior, see {@link CryptographicHashAttribute}.
|
||||
*
|
||||
* <p>
|
||||
* This processor identifies groups of user-specified flowfile attributes and assigns a unique hash value to each group, recording this hash value in the flowfile's attributes using a user-specified
|
||||
* attribute key. The groups are identified dynamically and preserved across application restarts. </p>
|
||||
*
|
||||
* <p>
|
||||
* The user must supply optional processor properties during runtime to correctly configure this processor. The optional property key will be used as the flowfile attribute key for attribute
|
||||
* inspection. The value must be a valid regular expression. This regular expression is evaluated against the flowfile attribute values. If the regular expression contains a capturing group, the value
|
||||
* of that group will be used when comparing flow file attributes. Otherwise, the original flow file attribute's value will be used if and only if the value matches the given regular expression. </p>
|
||||
*
|
||||
* <p>
|
||||
* If a flowfile does not have an attribute entry for one or more processor configured values, then the flowfile is routed to failure. </p>
|
||||
*
|
||||
* <p>
|
||||
* An example hash value identification:
|
||||
*
|
||||
* Assume Processor Configured with Two Properties ("MDKey1" = ".*" and "MDKey2" = "(.).*").
|
||||
*
|
||||
* FlowFile 1 has the following attributes: MDKey1 = a MDKey2 = b
|
||||
*
|
||||
* and will be assigned to group 1 (since no groups exist yet)
|
||||
*
|
||||
* FlowFile 2 has the following attributes: MDKey1 = 1 MDKey2 = 2
|
||||
*
|
||||
* and will be assigned to group 2 (attribute keys do not match existing groups)
|
||||
*
|
||||
* FlowFile 3 has the following attributes: MDKey1 = a MDKey2 = z
|
||||
*
|
||||
* and will be assigned to group 3 (attribute keys do not match existing groups)
|
||||
*
|
||||
* FlowFile 4 has the following attribute: MDKey1 = a MDKey2 = bad
|
||||
*
|
||||
* and will be assigned to group 1 (because the value of MDKey1 has the regular expression ".*" applied to it, and that evaluates to the same as MDKey1 attribute of the first flow file. Similarly, the
|
||||
* capturing group for the MDKey2 property indicates that only the first character of the MDKey2 attribute must match, and the first character of MDKey2 for Flow File 1 and Flow File 4 are both 'b'.)
|
||||
*
|
||||
* FlowFile 5 has the following attributes: MDKey1 = a
|
||||
*
|
||||
* and will route to failure because it does not have MDKey2 entry in its attribute
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The following flowfile attributes are created or modified: <ul>
|
||||
* <li><b><group.id.attribute.key></b> - The hash value.</li> </ul> </p>
|
||||
*/
|
||||
@EventDriven
|
||||
@SideEffectFree
|
||||
@SupportsBatching(defaultDuration = DefaultRunDuration.TWENTY_FIVE_MILLIS)
|
||||
@Tags({"attributes", "hash"})
|
||||
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||
@CapabilityDescription("Hashes together the key/value pairs of several flowfile attributes and adds the hash as a new attribute. "
|
||||
+ "Optional properties are to be added such that the name of the property is the name of a flowfile attribute to consider "
|
||||
+ "and the value of the property is a regular expression that, if matched by the attribute value, will cause that attribute "
|
||||
+ "to be used as part of the hash. If the regular expression contains a capturing group, only the value of the capturing "
|
||||
+ "group will be used. " + "For a processor which accepts various attributes and generates a cryptographic hash of each, see \"CryptographicHashAttribute\". ")
|
||||
@WritesAttribute(attribute = "<Hash Value Attribute Key>", description = "This Processor adds an attribute whose value is the result of "
|
||||
+ "Hashing the existing flowfile attributes. The name of this attribute is specified by the <Hash Value Attribute Key> property.")
|
||||
@DynamicProperty(name = "A flowfile attribute key for attribute inspection", value = "A Regular Expression",
|
||||
description = "This regular expression is evaluated against the "
|
||||
+ "flowfile attribute values. If the regular expression contains a capturing "
|
||||
+ "group, the value of that group will be used when comparing flow file "
|
||||
+ "attributes. Otherwise, the original flow file attribute's value will be used "
|
||||
+ "if and only if the value matches the given regular expression.")
|
||||
public class HashAttribute extends AbstractProcessor {
|
||||
|
||||
public static final PropertyDescriptor HASH_VALUE_ATTRIBUTE = new PropertyDescriptor.Builder()
|
||||
.name("Hash Value Attribute Key")
|
||||
.displayName("Hash Value Attribute Key")
|
||||
.description("The name of the flowfile attribute where the hash value should be stored")
|
||||
.required(true)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
|
||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||
.name("success")
|
||||
.description("Used for flowfiles that have a hash value added")
|
||||
.build();
|
||||
public static final Relationship REL_FAILURE = new Relationship.Builder()
|
||||
.name("failure")
|
||||
.description("Used for flowfiles that are missing required attributes")
|
||||
.build();
|
||||
|
||||
private Set<Relationship> relationships;
|
||||
private List<PropertyDescriptor> properties;
|
||||
private final AtomicReference<Map<String, Pattern>> regexMapRef = new AtomicReference<>(Collections.emptyMap());
|
||||
|
||||
@Override
|
||||
protected void init(final ProcessorInitializationContext context) {
|
||||
final Set<Relationship> relationships = new HashSet<>();
|
||||
relationships.add(REL_FAILURE);
|
||||
relationships.add(REL_SUCCESS);
|
||||
this.relationships = Collections.unmodifiableSet(relationships);
|
||||
|
||||
final List<PropertyDescriptor> properties = new ArrayList<>();
|
||||
properties.add(HASH_VALUE_ATTRIBUTE);
|
||||
this.properties = Collections.unmodifiableList(properties);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Relationship> getRelationships() {
|
||||
return relationships;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
|
||||
return new PropertyDescriptor.Builder()
|
||||
.name(propertyDescriptorName).addValidator(StandardValidators.createRegexValidator(0, 1, false)).required(false).dynamic(true).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
|
||||
if (descriptor.isRequired()) {
|
||||
return;
|
||||
}
|
||||
|
||||
final Map<String, Pattern> patternMap = new HashMap<>(regexMapRef.get());
|
||||
if (newValue == null) {
|
||||
patternMap.remove(descriptor.getName());
|
||||
} else {
|
||||
if (newValue.equals(".*")) {
|
||||
patternMap.put(descriptor.getName(), null);
|
||||
} else {
|
||||
final Pattern pattern = Pattern.compile(newValue);
|
||||
patternMap.put(descriptor.getName(), pattern);
|
||||
}
|
||||
}
|
||||
|
||||
regexMapRef.set(Collections.unmodifiableMap(patternMap));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) {
|
||||
FlowFile flowFile = session.get();
|
||||
if (flowFile == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final Map<String, Pattern> patterns = regexMapRef.get();
|
||||
final ComponentLog logger = getLogger();
|
||||
|
||||
final SortedMap<String, String> attributes = getRelevantAttributes(flowFile, patterns);
|
||||
if (attributes.size() != patterns.size()) {
|
||||
final Set<String> wantedKeys = patterns.keySet();
|
||||
final Set<String> foundKeys = attributes.keySet();
|
||||
final StringBuilder missingKeys = new StringBuilder();
|
||||
for (final String wantedKey : wantedKeys) {
|
||||
if (!foundKeys.contains(wantedKey)) {
|
||||
missingKeys.append(wantedKey).append(" ");
|
||||
}
|
||||
}
|
||||
|
||||
logger.error("routing {} to 'failure' because of missing attributes: {}", new Object[]{flowFile, missingKeys.toString()});
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
} else {
|
||||
// create single string of attribute key/value pairs to use for group ID hash
|
||||
final StringBuilder hashableValue = new StringBuilder();
|
||||
for (final Map.Entry<String, String> entry : attributes.entrySet()) {
|
||||
hashableValue.append(entry.getKey());
|
||||
if (StringUtils.isBlank(entry.getValue())) {
|
||||
hashableValue.append("EMPTY");
|
||||
} else {
|
||||
hashableValue.append(entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
// create group ID
|
||||
final String hashValue = DigestUtils.md5Hex(hashableValue.toString());
|
||||
|
||||
logger.info("adding Hash Value {} to attributes for {} and routing to success", new Object[]{hashValue, flowFile});
|
||||
flowFile = session.putAttribute(flowFile, context.getProperty(HASH_VALUE_ATTRIBUTE).getValue(), hashValue);
|
||||
session.getProvenanceReporter().modifyAttributes(flowFile);
|
||||
session.transfer(flowFile, REL_SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
private SortedMap<String, String> getRelevantAttributes(final FlowFile flowFile, final Map<String, Pattern> patterns) {
|
||||
final SortedMap<String, String> attributeMap = new TreeMap<>();
|
||||
for (final Map.Entry<String, Pattern> entry : patterns.entrySet()) {
|
||||
final String attributeName = entry.getKey();
|
||||
final String attributeValue = flowFile.getAttribute(attributeName);
|
||||
if (attributeValue != null) {
|
||||
final Pattern pattern = entry.getValue();
|
||||
if (pattern == null) {
|
||||
attributeMap.put(attributeName, attributeValue);
|
||||
} else {
|
||||
final Matcher matcher = pattern.matcher(attributeValue);
|
||||
if (matcher.matches()) {
|
||||
if (matcher.groupCount() == 0) {
|
||||
attributeMap.put(attributeName, matcher.group(0));
|
||||
} else {
|
||||
attributeMap.put(attributeName, matcher.group(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return attributeMap;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.DigestOutputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.Security;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.apache.nifi.annotation.behavior.EventDriven;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.SupportsBatching;
|
||||
import org.apache.nifi.annotation.behavior.WritesAttribute;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.AbstractProcessor;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processor.io.InputStreamCallback;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
import org.apache.nifi.stream.io.NullOutputStream;
|
||||
import org.apache.nifi.stream.io.StreamUtils;
|
||||
|
||||
@Deprecated
|
||||
@DeprecationNotice(classNames = {"org.apache.nifi.processors.standard.CryptographicHashContent"}, reason = "This processor is deprecated and may be removed in future releases.")
|
||||
@EventDriven
|
||||
@SupportsBatching
|
||||
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||
@Tags({"hash", "content", "MD5", "SHA-1", "SHA-256"})
|
||||
@CapabilityDescription("Calculates a hash value for the Content of a FlowFile and puts that hash value on the FlowFile as an attribute whose name "
|
||||
+ "is determined by the <Hash Attribute Name> property. "
|
||||
+ "This processor did not provide a consistent offering of hash algorithms, and is now deprecated. For modern cryptographic hashing capabilities, see \"CryptographicHashContent\". ")
|
||||
@WritesAttribute(attribute = "<Hash Attribute Name>", description = "This Processor adds an attribute whose value is the result of Hashing the "
|
||||
+ "existing FlowFile content. The name of this attribute is specified by the <Hash Attribute Name> property")
|
||||
public class HashContent extends AbstractProcessor {
|
||||
|
||||
public static final PropertyDescriptor ATTRIBUTE_NAME = new PropertyDescriptor.Builder()
|
||||
.name("Hash Attribute Name")
|
||||
.description("The name of the FlowFile Attribute into which the Hash Value should be written. If the value already exists, it will be overwritten")
|
||||
.required(true)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.defaultValue("hash.value")
|
||||
.build();
|
||||
|
||||
public static final PropertyDescriptor HASH_ALGORITHM = new PropertyDescriptor.Builder()
|
||||
.name("Hash Algorithm")
|
||||
.description("Determines what hashing algorithm should be used to perform the hashing function")
|
||||
.required(true)
|
||||
.allowableValues(Security.getAlgorithms("MessageDigest"))
|
||||
.defaultValue("MD5")
|
||||
.build();
|
||||
|
||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||
.name("success")
|
||||
.description("FlowFiles that are process successfully will be sent to this relationship")
|
||||
.build();
|
||||
public static final Relationship REL_FAILURE = new Relationship.Builder()
|
||||
.name("failure")
|
||||
.description("Any FlowFile that cannot be processed successfully will be sent to this relationship without any attribute being added")
|
||||
.build();
|
||||
|
||||
private List<PropertyDescriptor> properties;
|
||||
private Set<Relationship> relationships;
|
||||
|
||||
@Override
|
||||
protected void init(ProcessorInitializationContext context) {
|
||||
final List<PropertyDescriptor> props = new ArrayList<>();
|
||||
props.add(ATTRIBUTE_NAME);
|
||||
props.add(HASH_ALGORITHM);
|
||||
properties = Collections.unmodifiableList(props);
|
||||
|
||||
final Set<Relationship> rels = new HashSet<>();
|
||||
rels.add(REL_SUCCESS);
|
||||
rels.add(REL_FAILURE);
|
||||
relationships = Collections.unmodifiableSet(rels);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Relationship> getRelationships() {
|
||||
return relationships;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
|
||||
FlowFile flowFile = session.get();
|
||||
if (flowFile == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final ComponentLog logger = getLogger();
|
||||
final String algorithm = context.getProperty(HASH_ALGORITHM).getValue();
|
||||
final MessageDigest digest;
|
||||
try {
|
||||
digest = MessageDigest.getInstance(algorithm);
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
logger.error("Failed to process {} due to {}; routing to failure", new Object[]{flowFile, e});
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
return;
|
||||
}
|
||||
|
||||
final AtomicReference<String> hashValueHolder = new AtomicReference<>(null);
|
||||
|
||||
try {
|
||||
session.read(flowFile, new InputStreamCallback() {
|
||||
@Override
|
||||
public void process(final InputStream in) throws IOException {
|
||||
try (final DigestOutputStream digestOut = new DigestOutputStream(new NullOutputStream(), digest)) {
|
||||
StreamUtils.copy(in, digestOut);
|
||||
|
||||
final byte[] hash = digest.digest();
|
||||
final StringBuilder strb = new StringBuilder(hash.length * 2);
|
||||
for (int i = 0; i < hash.length; i++) {
|
||||
strb.append(Integer.toHexString((hash[i] & 0xFF) | 0x100), 1, 3);
|
||||
}
|
||||
|
||||
hashValueHolder.set(strb.toString());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
final String attributeName = context.getProperty(ATTRIBUTE_NAME).getValue();
|
||||
flowFile = session.putAttribute(flowFile, attributeName, hashValueHolder.get());
|
||||
logger.info("Successfully added attribute '{}' to {} with a value of {}; routing to success", new Object[]{attributeName, flowFile, hashValueHolder.get()});
|
||||
session.getProvenanceReporter().modifyAttributes(flowFile);
|
||||
session.transfer(flowFile, REL_SUCCESS);
|
||||
} catch (final ProcessException e) {
|
||||
logger.error("Failed to process {} due to {}; routing to failure", new Object[]{flowFile, e});
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -95,8 +95,6 @@ import org.apache.nifi.components.PropertyDescriptor;
|
|||
import org.apache.nifi.components.ValidationContext;
|
||||
import org.apache.nifi.components.ValidationResult;
|
||||
import org.apache.nifi.context.PropertyContext;
|
||||
import org.apache.nifi.deprecation.log.DeprecationLogger;
|
||||
import org.apache.nifi.deprecation.log.DeprecationLoggerFactory;
|
||||
import org.apache.nifi.expression.AttributeExpression;
|
||||
import org.apache.nifi.expression.ExpressionLanguageScope;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
|
|
@ -262,59 +260,6 @@ public class InvokeHTTP extends AbstractProcessor {
|
|||
.addValidator(StandardValidators.INTEGER_VALIDATOR)
|
||||
.build();
|
||||
|
||||
@Deprecated
|
||||
public static final PropertyDescriptor PROXY_HOST = new PropertyDescriptor.Builder()
|
||||
.name("Proxy Host")
|
||||
.description("Proxy Host and dependent properties are deprecated in favor of Proxy Configuration Service. Proxy Host can be configured using an IP address or DNS address.")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.build();
|
||||
|
||||
@Deprecated
|
||||
public static final PropertyDescriptor PROXY_PORT = new PropertyDescriptor.Builder()
|
||||
.name("Proxy Port")
|
||||
.description("Proxy Port and dependent properties are deprecated in favor of Proxy Configuration Service. Port number for the configured Proxy Host address.")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.PORT_VALIDATOR)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.dependsOn(PROXY_HOST)
|
||||
.build();
|
||||
|
||||
@Deprecated
|
||||
public static final PropertyDescriptor PROXY_TYPE = new PropertyDescriptor.Builder()
|
||||
.name("Proxy Type")
|
||||
.displayName("Proxy Type")
|
||||
.description("Proxy Type and dependent properties are deprecated in favor of Proxy Configuration Service. Proxy protocol type is not used")
|
||||
.defaultValue("http")
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
|
||||
.dependsOn(PROXY_HOST)
|
||||
.build();
|
||||
|
||||
@Deprecated
|
||||
public static final PropertyDescriptor PROXY_USERNAME = new PropertyDescriptor.Builder()
|
||||
.name("invokehttp-proxy-user")
|
||||
.displayName("Proxy Username")
|
||||
.description("Proxy Username and dependent properties are deprecated in favor of Proxy Configuration Service. Username to set when authenticating with a Proxy server.")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.dependsOn(PROXY_HOST)
|
||||
.build();
|
||||
|
||||
@Deprecated
|
||||
public static final PropertyDescriptor PROXY_PASSWORD = new PropertyDescriptor.Builder()
|
||||
.name("invokehttp-proxy-password")
|
||||
.displayName("Proxy Password")
|
||||
.description("Proxy Password and dependent properties are deprecated in favor of Proxy Configuration Service. Password to set when authenticating with a Proxy server.")
|
||||
.required(false)
|
||||
.sensitive(true)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
|
||||
.dependsOn(PROXY_HOST)
|
||||
.build();
|
||||
|
||||
public static final PropertyDescriptor REQUEST_OAUTH2_ACCESS_TOKEN_PROVIDER = new PropertyDescriptor.Builder()
|
||||
.name("oauth2-access-token-provider")
|
||||
.displayName("Request OAuth2 Access Token Provider")
|
||||
|
|
@ -563,11 +508,6 @@ public class InvokeHTTP extends AbstractProcessor {
|
|||
SOCKET_IDLE_TIMEOUT,
|
||||
SOCKET_IDLE_CONNECTIONS,
|
||||
PROXY_CONFIGURATION_SERVICE,
|
||||
PROXY_HOST,
|
||||
PROXY_PORT,
|
||||
PROXY_TYPE,
|
||||
PROXY_USERNAME,
|
||||
PROXY_PASSWORD,
|
||||
REQUEST_OAUTH2_ACCESS_TOKEN_PROVIDER,
|
||||
REQUEST_USERNAME,
|
||||
REQUEST_PASSWORD,
|
||||
|
|
@ -632,8 +572,6 @@ public class InvokeHTTP extends AbstractProcessor {
|
|||
|
||||
private static final String MULTIPLE_HEADER_DELIMITER = ", ";
|
||||
|
||||
private static final DeprecationLogger deprecationLogger = DeprecationLoggerFactory.getLogger(InvokeHTTP.class);
|
||||
|
||||
private volatile Set<String> dynamicPropertyNames = new HashSet<>();
|
||||
|
||||
private volatile Pattern requestHeaderAttributesPattern = null;
|
||||
|
|
@ -706,33 +644,7 @@ public class InvokeHTTP extends AbstractProcessor {
|
|||
|
||||
@Override
|
||||
protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
|
||||
final List<ValidationResult> results = new ArrayList<>(3);
|
||||
final boolean proxyHostSet = validationContext.getProperty(PROXY_HOST).isSet();
|
||||
if (proxyHostSet) {
|
||||
deprecationLogger.warn("{}[id={}] [{}] Property should be replaced with [{}] Property",
|
||||
getClass().getSimpleName(),
|
||||
getIdentifier(),
|
||||
PROXY_HOST.getDisplayName(),
|
||||
PROXY_CONFIGURATION_SERVICE.getDisplayName()
|
||||
);
|
||||
}
|
||||
|
||||
final boolean proxyPortSet = validationContext.getProperty(PROXY_PORT).isSet();
|
||||
|
||||
if ((proxyHostSet && !proxyPortSet) || (!proxyHostSet && proxyPortSet)) {
|
||||
results.add(new ValidationResult.Builder().subject("Proxy Host and Port").valid(false).explanation("If Proxy Host or Proxy Port is set, both must be set").build());
|
||||
}
|
||||
|
||||
final boolean proxyUserSet = validationContext.getProperty(PROXY_USERNAME).isSet();
|
||||
final boolean proxyPwdSet = validationContext.getProperty(PROXY_PASSWORD).isSet();
|
||||
|
||||
if ((proxyUserSet && !proxyPwdSet) || (!proxyUserSet && proxyPwdSet)) {
|
||||
results.add(new ValidationResult.Builder().subject("Proxy User and Password").valid(false).explanation("If Proxy Username or Proxy Password is set, both must be set").build());
|
||||
}
|
||||
if (proxyUserSet && !proxyHostSet) {
|
||||
results.add(new ValidationResult.Builder().subject("Proxy").valid(false).explanation("If Proxy username is set, proxy host must be set").build());
|
||||
}
|
||||
|
||||
final List<ValidationResult> results = new ArrayList<>();
|
||||
ProxyConfiguration.validateProxySpec(validationContext, results, PROXY_SPECS);
|
||||
|
||||
// Check for dynamic properties for form components.
|
||||
|
|
@ -794,21 +706,7 @@ public class InvokeHTTP extends AbstractProcessor {
|
|||
|
||||
OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient().newBuilder();
|
||||
|
||||
final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context, () -> {
|
||||
final ProxyConfiguration componentProxyConfig = new ProxyConfiguration();
|
||||
final String proxyHost = context.getProperty(PROXY_HOST).evaluateAttributeExpressions().getValue();
|
||||
final Integer proxyPort = context.getProperty(PROXY_PORT).evaluateAttributeExpressions().asInteger();
|
||||
if (proxyHost != null && proxyPort != null) {
|
||||
componentProxyConfig.setProxyType(Type.HTTP);
|
||||
componentProxyConfig.setProxyServerHost(proxyHost);
|
||||
componentProxyConfig.setProxyServerPort(proxyPort);
|
||||
final String proxyUsername = trimToEmpty(context.getProperty(PROXY_USERNAME).evaluateAttributeExpressions().getValue());
|
||||
final String proxyPassword = context.getProperty(PROXY_PASSWORD).evaluateAttributeExpressions().getValue();
|
||||
componentProxyConfig.setProxyUserName(proxyUsername);
|
||||
componentProxyConfig.setProxyUserPassword(proxyPassword);
|
||||
}
|
||||
return componentProxyConfig;
|
||||
});
|
||||
final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context);
|
||||
|
||||
final Proxy proxy = proxyConfig.createProxy();
|
||||
if (!Type.DIRECT.equals(proxy.type())) {
|
||||
|
|
|
|||
|
|
@ -1,993 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import static org.apache.nifi.processors.standard.util.HTTPUtils.PROXY_HOST;
|
||||
import static org.apache.nifi.processors.standard.util.HTTPUtils.PROXY_PORT;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.security.Principal;
|
||||
import java.security.cert.Certificate;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.regex.Pattern;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLPeerUnverifiedException;
|
||||
import javax.net.ssl.SSLSession;
|
||||
import javax.security.auth.x500.X500Principal;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.ws.rs.core.Response.Status;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpException;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpResponseInterceptor;
|
||||
import org.apache.http.NoHttpResponseException;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.client.HttpRequestRetryHandler;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpDelete;
|
||||
import org.apache.http.client.methods.HttpHead;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.config.Registry;
|
||||
import org.apache.http.config.RegistryBuilder;
|
||||
import org.apache.http.config.SocketConfig;
|
||||
import org.apache.http.conn.ManagedHttpClientConnection;
|
||||
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
||||
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.entity.ContentProducer;
|
||||
import org.apache.http.entity.EntityTemplate;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
import org.apache.http.protocol.HttpCoreContext;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.http.util.VersionInfo;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.behavior.SupportsBatching;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.annotation.lifecycle.OnScheduled;
|
||||
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.components.ValidationContext;
|
||||
import org.apache.nifi.components.ValidationResult;
|
||||
import org.apache.nifi.expression.ExpressionLanguageScope;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
import org.apache.nifi.flowfile.attributes.CoreAttributes;
|
||||
import org.apache.nifi.flowfile.attributes.StandardFlowFileMediaType;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.AbstractProcessor;
|
||||
import org.apache.nifi.processor.DataUnit;
|
||||
import org.apache.nifi.processor.FlowFileFilter;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processor.io.InputStreamCallback;
|
||||
import org.apache.nifi.processor.util.StandardValidators;
|
||||
import org.apache.nifi.processors.standard.util.HTTPUtils;
|
||||
import org.apache.nifi.security.util.CertificateUtils;
|
||||
import org.apache.nifi.ssl.SSLContextService;
|
||||
import org.apache.nifi.stream.io.GZIPOutputStream;
|
||||
import org.apache.nifi.stream.io.LeakyBucketStreamThrottler;
|
||||
import org.apache.nifi.stream.io.StreamThrottler;
|
||||
import org.apache.nifi.stream.io.StreamUtils;
|
||||
import org.apache.nifi.util.FlowFilePackager;
|
||||
import org.apache.nifi.util.FlowFilePackagerV1;
|
||||
import org.apache.nifi.util.FlowFilePackagerV2;
|
||||
import org.apache.nifi.util.FlowFilePackagerV3;
|
||||
import org.apache.nifi.util.FormatUtils;
|
||||
import org.apache.nifi.util.StopWatch;
|
||||
import org.apache.nifi.util.StringUtils;
|
||||
|
||||
@Deprecated
|
||||
@DeprecationNotice(alternatives = {InvokeHTTP.class}, reason = "This processor is deprecated and may be removed in future releases.")
|
||||
@SupportsBatching
|
||||
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||
@Tags({"http", "https", "remote", "copy", "archive"})
|
||||
@CapabilityDescription("Please be aware this processor is deprecated and may be removed in the near future. Use InvokeHTTP instead. Performs an HTTP Post with the content of the FlowFile. "
|
||||
+ "Uses a connection pool with max number of connections equal to "
|
||||
+ "the number of possible endpoints multiplied by the Concurrent Tasks configuration.")
|
||||
public class PostHTTP extends AbstractProcessor {
|
||||
|
||||
public static final String CONTENT_TYPE_HEADER = "Content-Type";
|
||||
public static final String ACCEPT = "Accept";
|
||||
public static final String ACCEPT_ENCODING = "Accept-Encoding";
|
||||
public static final String DEFAULT_CONTENT_TYPE = "application/octet-stream";
|
||||
public static final String FLOWFILE_CONFIRMATION_HEADER = "x-prefer-acknowledge-uri";
|
||||
public static final String LOCATION_HEADER_NAME = "Location";
|
||||
public static final String LOCATION_URI_INTENT_NAME = "x-location-uri-intent";
|
||||
public static final String LOCATION_URI_INTENT_VALUE = "flowfile-hold";
|
||||
public static final String GZIPPED_HEADER = "flowfile-gzipped";
|
||||
public static final String CONTENT_ENCODING_HEADER = "Content-Encoding";
|
||||
public static final String CONTENT_ENCODING_GZIP_VALUE = "gzip";
|
||||
|
||||
public static final String PROTOCOL_VERSION_HEADER = "x-nifi-transfer-protocol-version";
|
||||
public static final String TRANSACTION_ID_HEADER = "x-nifi-transaction-id";
|
||||
public static final String PROTOCOL_VERSION = "3";
|
||||
public static final String REMOTE_DN = "remote.dn";
|
||||
|
||||
private static final String FLOW_FILE_CONNECTION_LOG = "Connection to URI {} will be using Content Type {} if sending data as FlowFile";
|
||||
|
||||
public static final PropertyDescriptor URL = new PropertyDescriptor.Builder()
|
||||
.name("URL")
|
||||
.description("The URL to POST to. The URL may be defined using the Attribute Expression Language. "
|
||||
+ "A separate connection pool will be created for each unique host:port combination.")
|
||||
.required(true)
|
||||
.addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("https?\\://.*")))
|
||||
.addValidator(StandardValidators.URL_VALIDATOR)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
|
||||
.build();
|
||||
public static final PropertyDescriptor SEND_AS_FLOWFILE = new PropertyDescriptor.Builder()
|
||||
.name("Send as FlowFile")
|
||||
.description("If true, will package the FlowFile's contents and attributes together and send the FlowFile Package; otherwise, will send only the FlowFile's content")
|
||||
.required(true)
|
||||
.allowableValues("true", "false")
|
||||
.defaultValue("false")
|
||||
.build();
|
||||
public static final PropertyDescriptor CONNECTION_TIMEOUT = new PropertyDescriptor.Builder()
|
||||
.name("Connection Timeout")
|
||||
.description("How long to wait when attempting to connect to the remote server before giving up")
|
||||
.required(true)
|
||||
.defaultValue("30 sec")
|
||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor DATA_TIMEOUT = new PropertyDescriptor.Builder()
|
||||
.name("Data Timeout")
|
||||
.description("How long to wait between receiving segments of data from the remote server before giving up and discarding the partial file")
|
||||
.required(true)
|
||||
.defaultValue("30 sec")
|
||||
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder()
|
||||
.name("Username")
|
||||
.description("Username required to access the URL")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder()
|
||||
.name("Password")
|
||||
.description("Password required to access the URL")
|
||||
.required(false)
|
||||
.sensitive(true)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor USER_AGENT = new PropertyDescriptor.Builder()
|
||||
.name("User Agent")
|
||||
.description("What to report as the User Agent when we connect to the remote server")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.defaultValue(VersionInfo.getUserAgent("Apache-HttpClient", "org.apache.http.client", HttpClientBuilder.class))
|
||||
.build();
|
||||
public static final PropertyDescriptor COMPRESSION_LEVEL = new PropertyDescriptor.Builder()
|
||||
.name("Compression Level")
|
||||
.description("Determines the GZIP Compression Level to use when sending the file; the value must be in the range of 0-9. A value of 0 indicates that the file will not be GZIP'ed")
|
||||
.required(true)
|
||||
.addValidator(StandardValidators.createLongValidator(0, 9, true))
|
||||
.defaultValue("0")
|
||||
.build();
|
||||
public static final PropertyDescriptor ATTRIBUTES_AS_HEADERS_REGEX = new PropertyDescriptor.Builder()
|
||||
.name("Attributes to Send as HTTP Headers (Regex)")
|
||||
.description("Specifies the Regular Expression that determines the names of FlowFile attributes that should be sent as HTTP Headers")
|
||||
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
|
||||
.required(false)
|
||||
.build();
|
||||
public static final PropertyDescriptor MAX_DATA_RATE = new PropertyDescriptor.Builder()
|
||||
.name("Max Data to Post per Second")
|
||||
.description("The maximum amount of data to send per second; this allows the bandwidth to be throttled to a specified data rate; if not specified, the data rate is not throttled")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
|
||||
.build();
|
||||
public static final PropertyDescriptor MAX_BATCH_SIZE = new PropertyDescriptor.Builder()
|
||||
.name("Max Batch Size")
|
||||
.description("If the Send as FlowFile property is true, specifies the max data size for a batch of FlowFiles to send in a single "
|
||||
+ "HTTP POST. If not specified, each FlowFile will be sent separately. If the Send as FlowFile property is false, this "
|
||||
+ "property is ignored")
|
||||
.required(false)
|
||||
.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
|
||||
.defaultValue("100 MB")
|
||||
.build();
|
||||
public static final PropertyDescriptor CHUNKED_ENCODING = new PropertyDescriptor.Builder()
|
||||
.name("Use Chunked Encoding")
|
||||
.description("Specifies whether or not to use Chunked Encoding to send the data. This property is ignored in the event the contents are compressed "
|
||||
+ "or sent as FlowFiles.")
|
||||
.allowableValues("true", "false")
|
||||
.build();
|
||||
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||
.name("SSL Context Service")
|
||||
.description("The Controller Service to use in order to obtain an SSL Context")
|
||||
.required(false)
|
||||
.identifiesControllerService(SSLContextService.class)
|
||||
.build();
|
||||
public static final PropertyDescriptor CONTENT_TYPE = new PropertyDescriptor.Builder()
|
||||
.name("Content-Type")
|
||||
.description("The Content-Type to specify for the content of the FlowFile being POSTed if " + SEND_AS_FLOWFILE.getName() + " is false. "
|
||||
+ "In the case of an empty value after evaluating an expression language expression, Content-Type defaults to " + DEFAULT_CONTENT_TYPE)
|
||||
.required(true)
|
||||
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
|
||||
.defaultValue("${" + CoreAttributes.MIME_TYPE.key() + "}")
|
||||
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||
.build();
|
||||
|
||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||
.name("success")
|
||||
.description("Files that are successfully send will be transferred to success")
|
||||
.build();
|
||||
public static final Relationship REL_FAILURE = new Relationship.Builder()
|
||||
.name("failure")
|
||||
.description("Files that fail to send will transferred to failure")
|
||||
.build();
|
||||
|
||||
private Set<Relationship> relationships;
|
||||
private List<PropertyDescriptor> properties;
|
||||
|
||||
private final AtomicReference<StreamThrottler> throttlerRef = new AtomicReference<>();
|
||||
private final ConcurrentMap<String, DestinationAccepts> destinationAcceptsMap = new ConcurrentHashMap<>();
|
||||
private volatile PoolingHttpClientConnectionManager connManager;
|
||||
private volatile CloseableHttpClient client;
|
||||
private volatile RequestConfig requestConfig;
|
||||
|
||||
// this is used when creating thet HttpContext, which is a thread local variable that is used by
|
||||
// HTTPClient to obtain an available, reusable connection
|
||||
private volatile Principal principal;
|
||||
|
||||
@Override
|
||||
protected void init(final ProcessorInitializationContext context) {
|
||||
final Set<Relationship> relationships = new HashSet<>();
|
||||
relationships.add(REL_SUCCESS);
|
||||
relationships.add(REL_FAILURE);
|
||||
this.relationships = Collections.unmodifiableSet(relationships);
|
||||
|
||||
final List<PropertyDescriptor> properties = new ArrayList<>();
|
||||
properties.add(URL);
|
||||
properties.add(MAX_BATCH_SIZE);
|
||||
properties.add(MAX_DATA_RATE);
|
||||
properties.add(SSL_CONTEXT_SERVICE);
|
||||
properties.add(USERNAME);
|
||||
properties.add(PASSWORD);
|
||||
properties.add(SEND_AS_FLOWFILE);
|
||||
properties.add(CHUNKED_ENCODING);
|
||||
properties.add(COMPRESSION_LEVEL);
|
||||
properties.add(CONNECTION_TIMEOUT);
|
||||
properties.add(DATA_TIMEOUT);
|
||||
properties.add(ATTRIBUTES_AS_HEADERS_REGEX);
|
||||
properties.add(USER_AGENT);
|
||||
properties.add(HTTPUtils.PROXY_CONFIGURATION_SERVICE);
|
||||
properties.add(PROXY_HOST);
|
||||
properties.add(PROXY_PORT);
|
||||
properties.add(CONTENT_TYPE);
|
||||
this.properties = Collections.unmodifiableList(properties);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Relationship> getRelationships() {
|
||||
return relationships;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<ValidationResult> customValidate(final ValidationContext context) {
|
||||
final Collection<ValidationResult> results = new ArrayList<>();
|
||||
|
||||
if (context.getProperty(URL).getValue().startsWith("https") && context.getProperty(SSL_CONTEXT_SERVICE).getValue() == null) {
|
||||
results.add(new ValidationResult.Builder()
|
||||
.explanation("URL is set to HTTPS protocol but no SSLContext has been specified")
|
||||
.valid(false).subject("SSL Context").build());
|
||||
}
|
||||
|
||||
boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
|
||||
int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
|
||||
boolean chunkedSet = context.getProperty(CHUNKED_ENCODING).isSet();
|
||||
|
||||
if (compressionLevel == 0 && !sendAsFlowFile && !chunkedSet) {
|
||||
results.add(new ValidationResult.Builder().valid(false).subject(CHUNKED_ENCODING.getName())
|
||||
.explanation("if compression level is 0 and not sending as a FlowFile, then the \'" + CHUNKED_ENCODING.getName() + "\' property must be set").build());
|
||||
}
|
||||
|
||||
HTTPUtils.validateProxyProperties(context, results);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@OnStopped
|
||||
public void onStopped() {
|
||||
destinationAcceptsMap.clear();
|
||||
|
||||
try {
|
||||
connManager.shutdown();
|
||||
client.close();
|
||||
} catch (IOException e) {
|
||||
getLogger().error("Could not properly shutdown connections", e);
|
||||
}
|
||||
|
||||
final StreamThrottler throttler = throttlerRef.getAndSet(null);
|
||||
if (throttler != null) {
|
||||
try {
|
||||
throttler.close();
|
||||
} catch (IOException e) {
|
||||
getLogger().error("Failed to close StreamThrottler", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OnScheduled
|
||||
public void onScheduled(final ProcessContext context) {
|
||||
final Double bytesPerSecond = context.getProperty(MAX_DATA_RATE).asDataSize(DataUnit.B);
|
||||
this.throttlerRef.set(bytesPerSecond == null ? null : new LeakyBucketStreamThrottler(bytesPerSecond.intValue()));
|
||||
|
||||
String hostname = "unknown";
|
||||
try {
|
||||
hostname = InetAddress.getLocalHost().getCanonicalHostName();
|
||||
} catch (UnknownHostException ignore) {}
|
||||
principal = new X500Principal("CN=" + hostname + ", OU=unknown, O=unknown, C=unknown");
|
||||
|
||||
// setup the PoolingHttpClientConnectionManager
|
||||
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
|
||||
if (sslContextService == null) {
|
||||
connManager = new PoolingHttpClientConnectionManager();
|
||||
|
||||
} else {
|
||||
final SSLContext sslContext;
|
||||
try {
|
||||
sslContext = sslContextService.createContext();
|
||||
} catch (final Exception e) {
|
||||
throw new ProcessException(e);
|
||||
}
|
||||
|
||||
final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
|
||||
// Also use a plain socket factory for regular http connections (especially proxies)
|
||||
final Registry<ConnectionSocketFactory> socketFactoryRegistry =
|
||||
RegistryBuilder.<ConnectionSocketFactory>create()
|
||||
.register("https", sslsf)
|
||||
.register("http", PlainConnectionSocketFactory.getSocketFactory())
|
||||
.build();
|
||||
|
||||
connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
||||
}
|
||||
|
||||
// setup SocketConfig
|
||||
SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
|
||||
socketConfigBuilder.setSoTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
SocketConfig socketConfig = socketConfigBuilder.build();
|
||||
connManager.setDefaultSocketConfig(socketConfig);
|
||||
|
||||
// the +1 here accommodates math error calculating excess connections in AbstractConnPool.getPoolEntryBlocking()
|
||||
connManager.setDefaultMaxPerRoute(context.getMaxConcurrentTasks() + 1);
|
||||
// max total connections will get set in onTrigger(), because a new route will require increasing this
|
||||
connManager.setMaxTotal(1);
|
||||
// enable inactivity check, to detect and close idle connections
|
||||
connManager.setValidateAfterInactivity(30_000);
|
||||
|
||||
// setup the HttpClientBuilder
|
||||
final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
|
||||
clientBuilder.setConnectionManager(connManager);
|
||||
clientBuilder.setUserAgent(context.getProperty(USER_AGENT).getValue());
|
||||
clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
|
||||
@Override
|
||||
public void process(final HttpResponse response, final HttpContext httpContext) throws HttpException, IOException {
|
||||
final HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
|
||||
final ManagedHttpClientConnection conn = coreContext.getConnection(ManagedHttpClientConnection.class);
|
||||
if (!conn.isOpen()) {
|
||||
return;
|
||||
}
|
||||
|
||||
final SSLSession sslSession = conn.getSSLSession();
|
||||
|
||||
if (sslSession != null) {
|
||||
final Certificate[] certChain = sslSession.getPeerCertificates();
|
||||
if (certChain == null || certChain.length == 0) {
|
||||
throw new SSLPeerUnverifiedException("No certificates found");
|
||||
}
|
||||
|
||||
try {
|
||||
final X509Certificate cert = CertificateUtils.convertAbstractX509Certificate(certChain[0]);
|
||||
httpContext.setAttribute(REMOTE_DN, cert.getSubjectDN().getName().trim());
|
||||
} catch (CertificateException e) {
|
||||
final String msg = "Could not extract subject DN from SSL session peer certificate";
|
||||
getLogger().warn(msg);
|
||||
throw new SSLPeerUnverifiedException(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
HttpRequestRetryHandler retryHandler = (exception, attempt, httpContext) -> {
|
||||
if (attempt > 3 || !isScheduled()) {
|
||||
return false;
|
||||
}
|
||||
final HttpClientContext clientContext = HttpClientContext.adapt(httpContext);
|
||||
// A heavily loaded remote listener can manifest as NoHttpResponseExceptions here.
|
||||
// When this happens, take a 5 second snooze before retrying to give the remote a short break.
|
||||
if (exception instanceof NoHttpResponseException) {
|
||||
if (getLogger().isDebugEnabled()) {
|
||||
getLogger().debug("Sleeping for 5 secs then retrying {} request for remote server {}",
|
||||
new Object[]{clientContext.getRequest().getRequestLine().getMethod(), clientContext.getTargetHost()});
|
||||
}
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// do not retry more serious exceptions
|
||||
return false;
|
||||
};
|
||||
clientBuilder.setRetryHandler(retryHandler);
|
||||
clientBuilder.disableContentCompression();
|
||||
|
||||
final String username = context.getProperty(USERNAME).getValue();
|
||||
final String password = context.getProperty(PASSWORD).getValue();
|
||||
// set the credentials if appropriate
|
||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
if (username != null) {
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
|
||||
}
|
||||
|
||||
// Set the proxy if specified
|
||||
HTTPUtils.setProxy(context, clientBuilder, credentialsProvider);
|
||||
|
||||
// complete the HTTPClient build
|
||||
client = clientBuilder.build();
|
||||
|
||||
// setup RequestConfig
|
||||
final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
|
||||
requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
requestConfigBuilder.setRedirectsEnabled(false);
|
||||
requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
|
||||
requestConfig = requestConfigBuilder.build();
|
||||
}
|
||||
|
||||
private String getBaseUrl(final String url) {
|
||||
final int index = url.indexOf("/", 9);
|
||||
if (index < 0) {
|
||||
return url;
|
||||
}
|
||||
return url.substring(0, index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) {
|
||||
FlowFile firstFlowFile = session.get();
|
||||
if (firstFlowFile == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final ComponentLog logger = getLogger();
|
||||
final String url = context.getProperty(URL).evaluateAttributeExpressions(firstFlowFile).getValue();
|
||||
try {
|
||||
new java.net.URL(url);
|
||||
} catch (final MalformedURLException e) {
|
||||
logger.error("After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
|
||||
new Object[]{firstFlowFile, url});
|
||||
firstFlowFile = session.penalize(firstFlowFile);
|
||||
session.transfer(firstFlowFile, REL_FAILURE);
|
||||
return;
|
||||
}
|
||||
|
||||
final List<FlowFile> toSend = new ArrayList<>();
|
||||
toSend.add(firstFlowFile);
|
||||
|
||||
final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
|
||||
final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
|
||||
|
||||
final StreamThrottler throttler = throttlerRef.get();
|
||||
|
||||
final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
|
||||
final AtomicLong bytesToSend = new AtomicLong(firstFlowFile.getSize());
|
||||
|
||||
DestinationAccepts destinationAccepts = null;
|
||||
final String transactionId = UUID.randomUUID().toString();
|
||||
final HttpClientContext httpClientContext = HttpClientContext.create();
|
||||
httpClientContext.setUserToken(principal);
|
||||
|
||||
// determine whether or not destination accepts flowfile/gzip
|
||||
final String baseUrl = getBaseUrl(url);
|
||||
destinationAccepts = destinationAcceptsMap.get(baseUrl);
|
||||
if (destinationAccepts == null) {
|
||||
try {
|
||||
destinationAccepts = getDestinationAcceptance(sendAsFlowFile, url, transactionId, httpClientContext);
|
||||
if (null == destinationAcceptsMap.putIfAbsent(baseUrl, destinationAccepts)) {
|
||||
// url indicates a new route, so increase the max allowed open connections
|
||||
connManager.setMaxTotal(connManager.getMaxTotal() + connManager.getDefaultMaxPerRoute());
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
firstFlowFile = session.penalize(firstFlowFile);
|
||||
session.transfer(firstFlowFile, REL_FAILURE);
|
||||
logger.error("Unable to communicate with destination {} to determine whether or not it can accept "
|
||||
+ "flowfiles/gzip; routing {} to failure due to {}", new Object[]{url, firstFlowFile, e});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// if we are sending as flowfile and the destination accepts V3 or V2 (streaming) format,
|
||||
// then we can get more flowfiles from the session up to MAX_BATCH_SIZE for the same URL
|
||||
if (sendAsFlowFile && (destinationAccepts.isFlowFileV3Accepted() || destinationAccepts.isFlowFileV2Accepted())) {
|
||||
toSend.addAll(session.get(new FlowFileFilter() {
|
||||
@Override
|
||||
public FlowFileFilterResult filter(FlowFile flowFile) {
|
||||
// if over MAX_BATCH_SIZE, then stop adding files
|
||||
if (bytesToSend.get() + flowFile.getSize() > maxBatchBytes) {
|
||||
return FlowFileFilterResult.REJECT_AND_TERMINATE;
|
||||
}
|
||||
// check URL to see if this flowfile can be included in the batch
|
||||
final String urlToCheck = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
|
||||
if (url.equals(urlToCheck)) {
|
||||
bytesToSend.addAndGet(flowFile.getSize());
|
||||
return FlowFileFilterResult.ACCEPT_AND_CONTINUE;
|
||||
} else {
|
||||
return FlowFileFilterResult.REJECT_AND_CONTINUE;
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
final HttpPost post = new HttpPost(url);
|
||||
final DestinationAccepts accepts = destinationAccepts;
|
||||
final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;
|
||||
|
||||
final EntityTemplate entity = new EntityTemplate(new ContentProducer() {
|
||||
@Override
|
||||
public void writeTo(final OutputStream rawOut) throws IOException {
|
||||
final OutputStream throttled = throttler == null ? rawOut : throttler.newThrottledOutputStream(rawOut);
|
||||
OutputStream wrappedOut = new BufferedOutputStream(throttled);
|
||||
if (compressionLevel > 0 && accepts.isGzipAccepted()) {
|
||||
wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
|
||||
}
|
||||
|
||||
try (final OutputStream out = wrappedOut) {
|
||||
final FlowFilePackager packager;
|
||||
if (!sendAsFlowFile) {
|
||||
packager = null;
|
||||
} else if (accepts.isFlowFileV3Accepted()) {
|
||||
packager = new FlowFilePackagerV3();
|
||||
} else if (accepts.isFlowFileV2Accepted()) {
|
||||
packager = new FlowFilePackagerV2();
|
||||
} else if (accepts.isFlowFileV1Accepted()) {
|
||||
packager = new FlowFilePackagerV1();
|
||||
} else {
|
||||
packager = null;
|
||||
}
|
||||
|
||||
for (final FlowFile flowFile : toSend) {
|
||||
session.read(flowFile, new InputStreamCallback() {
|
||||
@Override
|
||||
public void process(final InputStream rawIn) throws IOException {
|
||||
try (final InputStream in = new BufferedInputStream(rawIn)) {
|
||||
|
||||
// if none of the above conditions is met, we should never get here, because
|
||||
// we will have already verified that at least 1 of the FlowFile packaging
|
||||
// formats is acceptable if sending as FlowFile.
|
||||
if (packager == null) {
|
||||
StreamUtils.copy(in, out);
|
||||
} else {
|
||||
final Map<String, String> flowFileAttributes;
|
||||
if (isDestinationLegacyNiFi) {
|
||||
// Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
|
||||
// in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
|
||||
flowFileAttributes = new HashMap<>(flowFile.getAttributes());
|
||||
flowFileAttributes.put("nf.file.name", flowFile.getAttribute(CoreAttributes.FILENAME.key()));
|
||||
flowFileAttributes.put("nf.file.path", flowFile.getAttribute(CoreAttributes.PATH.key()));
|
||||
} else {
|
||||
flowFileAttributes = flowFile.getAttributes();
|
||||
}
|
||||
|
||||
packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
out.flush();
|
||||
} catch (ProcessException pe) {
|
||||
// Pull out IOExceptions so that HTTPClient can properly do what it needs to do
|
||||
Throwable t = pe.getCause();
|
||||
if (t != null && t instanceof IOException) {
|
||||
IOException ioe = new IOException(t.getMessage());
|
||||
ioe.setStackTrace(t.getStackTrace());
|
||||
throw ioe;
|
||||
}
|
||||
throw pe;
|
||||
}
|
||||
}
|
||||
}) {
|
||||
|
||||
@Override
|
||||
public long getContentLength() {
|
||||
if (compressionLevel == 0 && !sendAsFlowFile && !context.getProperty(CHUNKED_ENCODING).asBoolean()) {
|
||||
return toSend.get(0).getSize();
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";
|
||||
|
||||
if (context.getProperty(CHUNKED_ENCODING).isSet()) {
|
||||
entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
|
||||
}
|
||||
post.setEntity(entity);
|
||||
post.setConfig(requestConfig);
|
||||
|
||||
final String contentType;
|
||||
if (sendAsFlowFile) {
|
||||
if (accepts.isFlowFileV3Accepted()) {
|
||||
contentType = StandardFlowFileMediaType.VERSION_3.getMediaType();
|
||||
} else if (accepts.isFlowFileV2Accepted()) {
|
||||
contentType = StandardFlowFileMediaType.VERSION_2.getMediaType();
|
||||
} else if (accepts.isFlowFileV1Accepted()) {
|
||||
contentType = StandardFlowFileMediaType.VERSION_1.getMediaType();
|
||||
} else {
|
||||
logger.error("Cannot send {} to {} because the destination does not accept FlowFiles and this processor is "
|
||||
+ "configured to deliver FlowFiles; routing to failure",
|
||||
new Object[] {flowFileDescription, url});
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
final String contentTypeValue = context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions(toSend.get(0)).getValue();
|
||||
contentType = StringUtils.isBlank(contentTypeValue) ? DEFAULT_CONTENT_TYPE : contentTypeValue;
|
||||
}
|
||||
|
||||
final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
|
||||
if (attributeHeaderRegex != null && !sendAsFlowFile && toSend.size() == 1) {
|
||||
final Pattern pattern = Pattern.compile(attributeHeaderRegex);
|
||||
|
||||
final Map<String, String> attributes = toSend.get(0).getAttributes();
|
||||
for (final Map.Entry<String, String> entry : attributes.entrySet()) {
|
||||
final String key = entry.getKey();
|
||||
if (pattern.matcher(key).matches()) {
|
||||
post.setHeader(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post.setHeader(CONTENT_TYPE_HEADER, contentType);
|
||||
post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
|
||||
post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
|
||||
post.setHeader(TRANSACTION_ID_HEADER, transactionId);
|
||||
if (compressionLevel > 0 && accepts.isGzipAccepted()) {
|
||||
if (sendAsFlowFile) {
|
||||
post.setHeader(GZIPPED_HEADER, "true");
|
||||
} else {
|
||||
post.setHeader(CONTENT_ENCODING_HEADER, CONTENT_ENCODING_GZIP_VALUE);
|
||||
}
|
||||
}
|
||||
|
||||
// Do the actual POST
|
||||
final String uploadDataRate;
|
||||
final long uploadMillis;
|
||||
CloseableHttpResponse response = null;
|
||||
try {
|
||||
final StopWatch stopWatch = new StopWatch(true);
|
||||
response = client.execute(post, httpClientContext);
|
||||
stopWatch.stop();
|
||||
uploadDataRate = stopWatch.calculateDataRate(bytesToSend.get());
|
||||
uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
|
||||
} catch (final IOException | ProcessException e) {
|
||||
logger.error("Failed to Post {} due to {}; transferring to failure", new Object[]{flowFileDescription, e});
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
return;
|
||||
} finally {
|
||||
if (response != null) {
|
||||
try {
|
||||
// consume input stream entirely, ignoring its contents. If we
|
||||
// don't do this, the Connection will not be returned to the pool
|
||||
EntityUtils.consume(response.getEntity());
|
||||
} catch (final IOException ignore) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
|
||||
// of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
|
||||
// from posting to some other webservice and then attempting to delete some resource to which
|
||||
// we are redirected
|
||||
final int responseCode = response.getStatusLine().getStatusCode();
|
||||
final String responseReason = response.getStatusLine().getReasonPhrase();
|
||||
String holdUri = null;
|
||||
if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
|
||||
final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
|
||||
if (locationUriHeader != null) {
|
||||
if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
|
||||
final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
|
||||
if (holdUriHeader != null) {
|
||||
holdUri = holdUriHeader.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (holdUri == null) {
|
||||
logger.error("Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
|
||||
new Object[]{flowFileDescription, url, responseCode, responseReason});
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (holdUri == null) {
|
||||
if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
|
||||
logger.error("Failed to Post {} to {}: response code was {}:{}",
|
||||
new Object[]{flowFileDescription, url, responseCode, responseReason});
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (responseCode >= 300) {
|
||||
logger.error("Failed to Post {} to {}: response code was {}:{}",
|
||||
new Object[]{flowFileDescription, url, responseCode, responseReason});
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info("Successfully Posted {} to {} in {} at a rate of {}",
|
||||
new Object[]{flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate});
|
||||
|
||||
for (final FlowFile flowFile : toSend) {
|
||||
session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + httpClientContext.getAttribute(REMOTE_DN), uploadMillis, true);
|
||||
session.transfer(flowFile, REL_SUCCESS);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
//
|
||||
// the response indicated a Hold URI; delete the Hold.
|
||||
//
|
||||
// determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
|
||||
// changed over the past, so we have to take into account a few different possibilities.
|
||||
String fullHoldUri = holdUri;
|
||||
if (holdUri.startsWith("/contentListener")) {
|
||||
// If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
|
||||
// as this really indicates that it should be whatever we posted to -- if posting directly to the
|
||||
// ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
|
||||
// be posting to some other URL.
|
||||
fullHoldUri = url + holdUri.substring(16);
|
||||
} else if (holdUri.startsWith("/")) {
|
||||
// URL indicates the full path but not hostname or port; use the same hostname & port that we posted
|
||||
// to but use the full path indicated by the response.
|
||||
int firstSlash = url.indexOf("/", 8);
|
||||
if (firstSlash < 0) {
|
||||
firstSlash = url.length();
|
||||
}
|
||||
final String beforeSlash = url.substring(0, firstSlash);
|
||||
fullHoldUri = beforeSlash + holdUri;
|
||||
} else if (!holdUri.startsWith("http")) {
|
||||
// Absolute URL
|
||||
fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
|
||||
}
|
||||
|
||||
final HttpDelete delete = new HttpDelete(fullHoldUri);
|
||||
delete.setHeader(TRANSACTION_ID_HEADER, transactionId);
|
||||
delete.setConfig(requestConfig);
|
||||
|
||||
HttpResponse holdResponse = null;
|
||||
try {
|
||||
holdResponse = client.execute(delete, httpClientContext);
|
||||
final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
|
||||
final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
|
||||
if (holdStatusCode >= 300) {
|
||||
logger.error("Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
|
||||
new Object[]{flowFileDescription, holdStatusCode, holdReason});
|
||||
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info("Successfully Posted {} to {} in {} at a rate of {}",
|
||||
new Object[]{flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate});
|
||||
|
||||
for (final FlowFile flowFile : toSend) {
|
||||
session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + httpClientContext.getAttribute(REMOTE_DN), uploadMillis, true);
|
||||
session.transfer(flowFile, REL_SUCCESS);
|
||||
}
|
||||
return;
|
||||
|
||||
} catch (final IOException e) {
|
||||
logger.warn("Failed to delete Hold that destination placed on {} due to {}; routing to failure", new Object[]{flowFileDescription, e});
|
||||
for (FlowFile flowFile : toSend) {
|
||||
flowFile = session.penalize(flowFile);
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
}
|
||||
} finally {
|
||||
if (null != holdResponse) {
|
||||
try {
|
||||
// consume input stream entirely, ignoring its contents. If we
|
||||
// don't do this, the Connection will not be returned to the pool
|
||||
EntityUtils.consume(holdResponse.getEntity());
|
||||
} catch (IOException ignore) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private DestinationAccepts getDestinationAcceptance(final boolean sendAsFlowFile, final String uri, final String transactionId, final HttpContext httpContext) throws IOException {
|
||||
final HttpHead head = new HttpHead(uri);
|
||||
head.setConfig(requestConfig);
|
||||
if (sendAsFlowFile) {
|
||||
head.addHeader(TRANSACTION_ID_HEADER, transactionId);
|
||||
}
|
||||
|
||||
final HttpResponse response = client.execute(head, httpContext);
|
||||
|
||||
// we assume that the destination can support FlowFile v1 always when the processor is also configured to send as a FlowFile
|
||||
// otherwise, we do not bother to make any determinations concerning this compatibility
|
||||
final boolean acceptsFlowFileV1 = sendAsFlowFile;
|
||||
boolean acceptsFlowFileV2 = false;
|
||||
boolean acceptsFlowFileV3 = false;
|
||||
boolean acceptsGzip = false;
|
||||
Integer protocolVersion = null;
|
||||
|
||||
final int statusCode = response.getStatusLine().getStatusCode();
|
||||
if (statusCode == Status.METHOD_NOT_ALLOWED.getStatusCode()) {
|
||||
return new DestinationAccepts(acceptsFlowFileV3, acceptsFlowFileV2, acceptsFlowFileV1, false, null);
|
||||
} else if (statusCode == Status.OK.getStatusCode()) {
|
||||
Header[] headers = response.getHeaders(ACCEPT);
|
||||
// If configured to send as a flowfile, determine the capabilities of the endpoint
|
||||
if (sendAsFlowFile) {
|
||||
if (headers != null) {
|
||||
for (final Header header : headers) {
|
||||
for (final String accepted : header.getValue().split(",")) {
|
||||
final String trimmed = accepted.trim();
|
||||
if (trimmed.equals(StandardFlowFileMediaType.VERSION_3.getMediaType())) {
|
||||
acceptsFlowFileV3 = true;
|
||||
} else if (trimmed.equals(StandardFlowFileMediaType.VERSION_2.getMediaType())) {
|
||||
acceptsFlowFileV2 = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final Header destinationVersion = response.getFirstHeader(PROTOCOL_VERSION_HEADER);
|
||||
if (destinationVersion != null) {
|
||||
try {
|
||||
protocolVersion = Integer.valueOf(destinationVersion.getValue());
|
||||
} catch (final NumberFormatException e) {
|
||||
// nothing to do here really.... it's an invalid value, so treat the same as if not specified
|
||||
}
|
||||
}
|
||||
|
||||
if (getLogger().isDebugEnabled()) {
|
||||
if (acceptsFlowFileV3) {
|
||||
getLogger().debug(FLOW_FILE_CONNECTION_LOG, new Object[]{uri, StandardFlowFileMediaType.VERSION_3.getMediaType()});
|
||||
} else if (acceptsFlowFileV2) {
|
||||
getLogger().debug(FLOW_FILE_CONNECTION_LOG, new Object[]{uri, StandardFlowFileMediaType.VERSION_2.getMediaType()});
|
||||
} else if (acceptsFlowFileV1) {
|
||||
getLogger().debug(FLOW_FILE_CONNECTION_LOG, new Object[]{uri, StandardFlowFileMediaType.VERSION_1.getMediaType()});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
headers = response.getHeaders(ACCEPT_ENCODING);
|
||||
if (headers != null) {
|
||||
for (final Header header : headers) {
|
||||
for (final String accepted : header.getValue().split(",")) {
|
||||
if (accepted.equalsIgnoreCase("gzip")) {
|
||||
acceptsGzip = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (getLogger().isDebugEnabled()) {
|
||||
if (acceptsGzip) {
|
||||
getLogger().debug("Connection to URI " + uri + " indicates that inline GZIP compression is supported");
|
||||
} else {
|
||||
getLogger().debug("Connection to URI " + uri + " indicates that it does NOT support inline GZIP compression");
|
||||
}
|
||||
}
|
||||
|
||||
return new DestinationAccepts(acceptsFlowFileV3, acceptsFlowFileV2, acceptsFlowFileV1, acceptsGzip, protocolVersion);
|
||||
} else {
|
||||
getLogger().warn("Unable to communicate with destination; when attempting to perform an HTTP HEAD, got unexpected response code of "
|
||||
+ statusCode + ": " + response.getStatusLine().getReasonPhrase());
|
||||
return new DestinationAccepts(false, false, false, false, null);
|
||||
}
|
||||
}
|
||||
|
||||
private static class DestinationAccepts {
|
||||
|
||||
private final boolean flowFileV1;
|
||||
private final boolean flowFileV2;
|
||||
private final boolean flowFileV3;
|
||||
private final boolean gzip;
|
||||
private final Integer protocolVersion;
|
||||
|
||||
public DestinationAccepts(final boolean flowFileV3, final boolean flowFileV2, final boolean flowFileV1, final boolean gzip, final Integer protocolVersion) {
|
||||
this.flowFileV3 = flowFileV3;
|
||||
this.flowFileV2 = flowFileV2;
|
||||
this.flowFileV1 = flowFileV1;
|
||||
this.gzip = gzip;
|
||||
this.protocolVersion = protocolVersion;
|
||||
}
|
||||
|
||||
public boolean isFlowFileV3Accepted() {
|
||||
return flowFileV3;
|
||||
}
|
||||
|
||||
public boolean isFlowFileV2Accepted() {
|
||||
return flowFileV2;
|
||||
}
|
||||
|
||||
public boolean isFlowFileV1Accepted() {
|
||||
return flowFileV1;
|
||||
}
|
||||
|
||||
public boolean isGzipAccepted() {
|
||||
return gzip;
|
||||
}
|
||||
|
||||
public Integer getProtocolVersion() {
|
||||
return protocolVersion;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,383 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.ATTRIBUTE_PREFIX;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.ATTRIBUTE_TYPE_SUFFIX;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_BOOLEAN;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_BYTE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_DOUBLE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_FLOAT;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_INTEGER;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_LONG;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_OBJECT;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_SHORT;
|
||||
import static org.apache.nifi.processors.standard.util.JmsFactory.PROP_TYPE_STRING;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.ATTRIBUTES_TO_JMS_PROPS;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.BATCH_SIZE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.CLIENT_ID_PREFIX;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.DESTINATION_NAME;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.DESTINATION_TYPE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.JMS_PROVIDER;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MAX_BUFFER_SIZE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MESSAGE_PRIORITY;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MESSAGE_TTL;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MESSAGE_TYPE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MSG_TYPE_BYTE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MSG_TYPE_EMPTY;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MSG_TYPE_MAP;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MSG_TYPE_STREAM;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.MSG_TYPE_TEXT;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.PASSWORD;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.REPLY_TO_QUEUE;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.TIMEOUT;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.URL;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.USERNAME;
|
||||
import static org.apache.nifi.processors.standard.util.JmsProperties.SSL_CONTEXT_SERVICE;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Queue;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.jms.BytesMessage;
|
||||
import javax.jms.Destination;
|
||||
import javax.jms.JMSException;
|
||||
import javax.jms.Message;
|
||||
import javax.jms.MessageProducer;
|
||||
import javax.jms.Session;
|
||||
import javax.jms.StreamMessage;
|
||||
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||
import org.apache.nifi.annotation.documentation.DeprecationNotice;
|
||||
import org.apache.nifi.annotation.documentation.SeeAlso;
|
||||
import org.apache.nifi.annotation.documentation.Tags;
|
||||
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
||||
import org.apache.nifi.components.PropertyDescriptor;
|
||||
import org.apache.nifi.flowfile.FlowFile;
|
||||
import org.apache.nifi.logging.ComponentLog;
|
||||
import org.apache.nifi.processor.AbstractProcessor;
|
||||
import org.apache.nifi.processor.DataUnit;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processor.exception.ProcessException;
|
||||
import org.apache.nifi.processor.io.InputStreamCallback;
|
||||
import org.apache.nifi.processors.standard.util.JmsFactory;
|
||||
import org.apache.nifi.processors.standard.util.WrappedMessageProducer;
|
||||
import org.apache.nifi.stream.io.StreamUtils;
|
||||
|
||||
@Deprecated
|
||||
@DeprecationNotice(classNames = {"org.apache.nifi.jms.processors.PublishJMS"}, reason = "This processor is deprecated and may be removed in future releases.")
|
||||
@Tags({"jms", "send", "put"})
|
||||
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||
@CapabilityDescription("Creates a JMS Message from the contents of a FlowFile and sends the message to a ActiveMQ JMS Server.")
|
||||
@SeeAlso({GetJMSQueue.class, GetJMSTopic.class, })
|
||||
public class PutJMS extends AbstractProcessor {
|
||||
|
||||
public static final Charset UTF8 = Charset.forName("UTF-8");
|
||||
public static final int DEFAULT_MESSAGE_PRIORITY = 4;
|
||||
|
||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||
.name("success")
|
||||
.description("All FlowFiles that are sent to the JMS destination are routed to this relationship")
|
||||
.build();
|
||||
public static final Relationship REL_FAILURE = new Relationship.Builder()
|
||||
.name("failure")
|
||||
.description("All FlowFiles that cannot be routed to the JMS destination are routed to this relationship")
|
||||
.build();
|
||||
|
||||
private final Queue<WrappedMessageProducer> producerQueue = new LinkedBlockingQueue<>();
|
||||
private final List<PropertyDescriptor> properties;
|
||||
private final Set<Relationship> relationships;
|
||||
|
||||
public PutJMS() {
|
||||
final List<PropertyDescriptor> descriptors = new ArrayList<>();
|
||||
descriptors.add(JMS_PROVIDER);
|
||||
descriptors.add(URL);
|
||||
descriptors.add(DESTINATION_NAME);
|
||||
descriptors.add(DESTINATION_TYPE);
|
||||
descriptors.add(TIMEOUT);
|
||||
descriptors.add(BATCH_SIZE);
|
||||
descriptors.add(USERNAME);
|
||||
descriptors.add(PASSWORD);
|
||||
descriptors.add(SSL_CONTEXT_SERVICE);
|
||||
descriptors.add(MESSAGE_TYPE);
|
||||
descriptors.add(MESSAGE_PRIORITY);
|
||||
descriptors.add(REPLY_TO_QUEUE);
|
||||
descriptors.add(MAX_BUFFER_SIZE);
|
||||
descriptors.add(MESSAGE_TTL);
|
||||
descriptors.add(ATTRIBUTES_TO_JMS_PROPS);
|
||||
descriptors.add(CLIENT_ID_PREFIX);
|
||||
this.properties = Collections.unmodifiableList(descriptors);
|
||||
|
||||
final Set<Relationship> relationships = new HashSet<>();
|
||||
relationships.add(REL_SUCCESS);
|
||||
relationships.add(REL_FAILURE);
|
||||
this.relationships = Collections.unmodifiableSet(relationships);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Relationship> getRelationships() {
|
||||
return relationships;
|
||||
}
|
||||
|
||||
@OnStopped
|
||||
public void cleanupResources() {
|
||||
WrappedMessageProducer wrappedProducer = producerQueue.poll();
|
||||
while (wrappedProducer != null) {
|
||||
wrappedProducer.close(getLogger());
|
||||
wrappedProducer = producerQueue.poll();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
|
||||
final ComponentLog logger = getLogger();
|
||||
final List<FlowFile> flowFiles = session.get(context.getProperty(BATCH_SIZE).asInteger().intValue());
|
||||
if (flowFiles.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
WrappedMessageProducer wrappedProducer = producerQueue.poll();
|
||||
if (wrappedProducer == null) {
|
||||
try {
|
||||
wrappedProducer = JmsFactory.createMessageProducer(context, true);
|
||||
logger.info("Connected to JMS server {}", new Object[]{context.getProperty(URL).getValue()});
|
||||
} catch (final JMSException e) {
|
||||
logger.error("Failed to connect to JMS Server due to {}", new Object[]{e});
|
||||
session.transfer(flowFiles, REL_FAILURE);
|
||||
context.yield();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
final Session jmsSession = wrappedProducer.getSession();
|
||||
final MessageProducer producer = wrappedProducer.getProducer();
|
||||
|
||||
final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
|
||||
|
||||
try {
|
||||
final Set<FlowFile> successfulFlowFiles = new HashSet<>();
|
||||
|
||||
for (FlowFile flowFile : flowFiles) {
|
||||
if (flowFile.getSize() > maxBufferSize) {
|
||||
session.transfer(flowFile, REL_FAILURE);
|
||||
logger.warn("Routing {} to failure because its size exceeds the configured max", new Object[]{flowFile});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Read the contents of the FlowFile into a byte array
|
||||
final byte[] messageContent = new byte[(int) flowFile.getSize()];
|
||||
session.read(flowFile, new InputStreamCallback() {
|
||||
@Override
|
||||
public void process(final InputStream in) throws IOException {
|
||||
StreamUtils.fillBuffer(in, messageContent, true);
|
||||
}
|
||||
});
|
||||
|
||||
final Long ttl = context.getProperty(MESSAGE_TTL).asTimePeriod(TimeUnit.MILLISECONDS);
|
||||
|
||||
final String replyToQueueName = context.getProperty(REPLY_TO_QUEUE).evaluateAttributeExpressions(flowFile).getValue();
|
||||
final Destination replyToQueue = replyToQueueName == null ? null : JmsFactory.createQueue(context, replyToQueueName);
|
||||
|
||||
int priority = DEFAULT_MESSAGE_PRIORITY;
|
||||
try {
|
||||
final Integer priorityInt = context.getProperty(MESSAGE_PRIORITY).evaluateAttributeExpressions(flowFile).asInteger();
|
||||
priority = priorityInt == null ? priority : priorityInt;
|
||||
} catch (final NumberFormatException e) {
|
||||
logger.warn("Invalid value for JMS Message Priority: {}; defaulting to priority of {}",
|
||||
new Object[]{context.getProperty(MESSAGE_PRIORITY).evaluateAttributeExpressions(flowFile).getValue(), DEFAULT_MESSAGE_PRIORITY});
|
||||
}
|
||||
|
||||
try {
|
||||
final Message message = createMessage(jmsSession, context, messageContent, flowFile, replyToQueue, priority);
|
||||
if (ttl == null) {
|
||||
producer.setTimeToLive(0L);
|
||||
} else {
|
||||
producer.setTimeToLive(ttl);
|
||||
}
|
||||
producer.send(message);
|
||||
} catch (final JMSException e) {
|
||||
logger.error("Failed to send {} to JMS Server due to {}", new Object[]{flowFile, e});
|
||||
session.transfer(flowFiles, REL_FAILURE);
|
||||
context.yield();
|
||||
|
||||
try {
|
||||
jmsSession.rollback();
|
||||
} catch (final JMSException jmse) {
|
||||
logger.warn("Unable to roll back JMS Session due to {}", new Object[]{jmse});
|
||||
}
|
||||
|
||||
wrappedProducer.close(logger);
|
||||
return;
|
||||
}
|
||||
|
||||
successfulFlowFiles.add(flowFile);
|
||||
session.getProvenanceReporter().send(flowFile, context.getProperty(URL).getValue());
|
||||
}
|
||||
|
||||
try {
|
||||
jmsSession.commit();
|
||||
|
||||
session.transfer(successfulFlowFiles, REL_SUCCESS);
|
||||
final String flowFileDescription = successfulFlowFiles.size() > 10 ? successfulFlowFiles.size() + " FlowFiles" : successfulFlowFiles.toString();
|
||||
logger.info("Sent {} to JMS Server and transferred to 'success'", new Object[]{flowFileDescription});
|
||||
} catch (JMSException e) {
|
||||
logger.error("Failed to commit JMS Session due to {} and transferred to 'failure'", new Object[]{e});
|
||||
session.transfer(flowFiles, REL_FAILURE);
|
||||
context.yield();
|
||||
wrappedProducer.close(logger);
|
||||
}
|
||||
} finally {
|
||||
if (!wrappedProducer.isClosed()) {
|
||||
producerQueue.offer(wrappedProducer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Message createMessage(final Session jmsSession, final ProcessContext context, final byte[] messageContent,
|
||||
final FlowFile flowFile, final Destination replyToQueue, final Integer priority) throws JMSException {
|
||||
final Message message;
|
||||
|
||||
switch (context.getProperty(MESSAGE_TYPE).getValue()) {
|
||||
case MSG_TYPE_EMPTY: {
|
||||
message = jmsSession.createTextMessage("");
|
||||
break;
|
||||
}
|
||||
case MSG_TYPE_STREAM: {
|
||||
final StreamMessage streamMessage = jmsSession.createStreamMessage();
|
||||
streamMessage.writeBytes(messageContent);
|
||||
message = streamMessage;
|
||||
break;
|
||||
}
|
||||
case MSG_TYPE_TEXT: {
|
||||
message = jmsSession.createTextMessage(new String(messageContent, UTF8));
|
||||
break;
|
||||
}
|
||||
case MSG_TYPE_MAP: {
|
||||
message = jmsSession.createMapMessage();
|
||||
break;
|
||||
}
|
||||
case MSG_TYPE_BYTE:
|
||||
default: {
|
||||
final BytesMessage bytesMessage = jmsSession.createBytesMessage();
|
||||
bytesMessage.writeBytes(messageContent);
|
||||
message = bytesMessage;
|
||||
}
|
||||
}
|
||||
|
||||
message.setJMSTimestamp(System.currentTimeMillis());
|
||||
|
||||
if (replyToQueue != null) {
|
||||
message.setJMSReplyTo(replyToQueue);
|
||||
}
|
||||
|
||||
if (priority != null) {
|
||||
message.setJMSPriority(priority);
|
||||
}
|
||||
|
||||
if (context.getProperty(ATTRIBUTES_TO_JMS_PROPS).asBoolean()) {
|
||||
copyAttributesToJmsProps(flowFile, message);
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterates through all of the flow file's metadata and for any metadata key that starts with <code>jms.</code>, the value for the corresponding key is written to the JMS message as a property.
|
||||
* The name of this property is equal to the key of the flow file's metadata minus the <code>jms.</code>. For example, if the flowFile has a metadata entry:
|
||||
* <br /><br />
|
||||
* <code>jms.count</code> = <code>8</code>
|
||||
* <br /><br />
|
||||
* then the JMS message will have a String property added to it with the property name <code>count</code> and value <code>8</code>.
|
||||
*
|
||||
* If the flow file also has a metadata key with the name <code>jms.count.type</code>, then the value of that metadata entry will determine the JMS property type to use for the value. For example,
|
||||
* if the flow file has the following properties:
|
||||
* <br /><br />
|
||||
* <code>jms.count</code> = <code>8</code><br />
|
||||
* <code>jms.count.type</code> = <code>integer</code>
|
||||
* <br /><br />
|
||||
* Then <code>message</code> will have an INTEGER property added with the value 8.
|
||||
* <br /><br/>
|
||||
* If the type is not valid for the given value (e.g., <code>jms.count.type</code> = <code>integer</code> and <code>jms.count</code> = <code>hello</code>, then this JMS property will not be added
|
||||
* to <code>message</code>.
|
||||
*
|
||||
* @param flowFile The flow file whose metadata should be examined for JMS properties.
|
||||
* @param message The JMS message to which we want to add properties.
|
||||
* @throws JMSException ex
|
||||
*/
|
||||
private void copyAttributesToJmsProps(final FlowFile flowFile, final Message message) throws JMSException {
|
||||
final ComponentLog logger = getLogger();
|
||||
|
||||
final Map<String, String> attributes = flowFile.getAttributes();
|
||||
for (final Entry<String, String> entry : attributes.entrySet()) {
|
||||
final String key = entry.getKey();
|
||||
final String value = entry.getValue();
|
||||
|
||||
if (key.toLowerCase().startsWith(ATTRIBUTE_PREFIX.toLowerCase()) && !key.toLowerCase().endsWith(ATTRIBUTE_TYPE_SUFFIX.toLowerCase())) {
|
||||
|
||||
final String jmsPropName = key.substring(ATTRIBUTE_PREFIX.length());
|
||||
final String type = attributes.get(key + ATTRIBUTE_TYPE_SUFFIX);
|
||||
|
||||
try {
|
||||
if (type == null || type.equalsIgnoreCase(PROP_TYPE_STRING)) {
|
||||
message.setStringProperty(jmsPropName, value);
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_INTEGER)) {
|
||||
message.setIntProperty(jmsPropName, Integer.parseInt(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_BOOLEAN)) {
|
||||
message.setBooleanProperty(jmsPropName, Boolean.parseBoolean(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_SHORT)) {
|
||||
message.setShortProperty(jmsPropName, Short.parseShort(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_LONG)) {
|
||||
message.setLongProperty(jmsPropName, Long.parseLong(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_BYTE)) {
|
||||
message.setByteProperty(jmsPropName, Byte.parseByte(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_DOUBLE)) {
|
||||
message.setDoubleProperty(jmsPropName, Double.parseDouble(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_FLOAT)) {
|
||||
message.setFloatProperty(jmsPropName, Float.parseFloat(value));
|
||||
} else if (type.equalsIgnoreCase(PROP_TYPE_OBJECT)) {
|
||||
message.setObjectProperty(jmsPropName, value);
|
||||
} else {
|
||||
logger.warn("Attribute key '{}' for {} has value '{}', but expected one of: integer, string, object, byte, double, float, long, short, boolean; not adding this property",
|
||||
new Object[]{key, flowFile, value});
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
logger.warn("Attribute key '{}' for {} has value '{}', but attribute key '{}' has value '{}'. Not adding this JMS property",
|
||||
new Object[]{key, flowFile, value, key + ATTRIBUTE_TYPE_SUFFIX, PROP_TYPE_INTEGER});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,7 +14,6 @@
|
|||
# limitations under the License.
|
||||
org.apache.nifi.processors.standard.AttributesToCSV
|
||||
org.apache.nifi.processors.standard.AttributesToJSON
|
||||
org.apache.nifi.processors.standard.Base64EncodeContent
|
||||
org.apache.nifi.processors.standard.CalculateRecordStats
|
||||
org.apache.nifi.processors.standard.CompressContent
|
||||
org.apache.nifi.processors.standard.ControlRate
|
||||
|
|
@ -53,14 +52,9 @@ org.apache.nifi.processors.standard.GenerateFlowFile
|
|||
org.apache.nifi.processors.standard.GenerateTableFetch
|
||||
org.apache.nifi.processors.standard.GetFile
|
||||
org.apache.nifi.processors.standard.GetFTP
|
||||
org.apache.nifi.processors.standard.GetHTTP
|
||||
org.apache.nifi.processors.standard.GetJMSQueue
|
||||
org.apache.nifi.processors.standard.GetJMSTopic
|
||||
org.apache.nifi.processors.standard.GetSFTP
|
||||
org.apache.nifi.processors.standard.HandleHttpRequest
|
||||
org.apache.nifi.processors.standard.HandleHttpResponse
|
||||
org.apache.nifi.processors.standard.HashAttribute
|
||||
org.apache.nifi.processors.standard.HashContent
|
||||
org.apache.nifi.processors.standard.IdentifyMimeType
|
||||
org.apache.nifi.processors.standard.InvokeHTTP
|
||||
org.apache.nifi.processors.standard.JoltTransformJSON
|
||||
|
|
@ -90,13 +84,11 @@ org.apache.nifi.processors.standard.ParseCEF
|
|||
org.apache.nifi.processors.standard.ParseSyslog
|
||||
org.apache.nifi.processors.standard.ParseSyslog5424
|
||||
org.apache.nifi.processors.standard.PartitionRecord
|
||||
org.apache.nifi.processors.standard.PostHTTP
|
||||
org.apache.nifi.processors.standard.PutDatabaseRecord
|
||||
org.apache.nifi.processors.standard.PutDistributedMapCache
|
||||
org.apache.nifi.processors.standard.PutEmail
|
||||
org.apache.nifi.processors.standard.PutFile
|
||||
org.apache.nifi.processors.standard.PutFTP
|
||||
org.apache.nifi.processors.standard.PutJMS
|
||||
org.apache.nifi.processors.standard.PutRecord
|
||||
org.apache.nifi.processors.standard.PutSFTP
|
||||
org.apache.nifi.processors.standard.PutSQL
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ import org.apache.nifi.processors.standard.http.FlowFileNamingStrategy;
|
|||
import org.apache.nifi.processors.standard.http.CookieStrategy;
|
||||
import org.apache.nifi.processors.standard.http.HttpHeader;
|
||||
import org.apache.nifi.processors.standard.http.HttpMethod;
|
||||
import org.apache.nifi.proxy.ProxyConfiguration;
|
||||
import org.apache.nifi.proxy.ProxyConfigurationService;
|
||||
import org.apache.nifi.reporting.InitializationException;
|
||||
import org.apache.nifi.security.util.StandardTlsConfiguration;
|
||||
import org.apache.nifi.security.util.TemporaryKeyStoreBuilder;
|
||||
|
|
@ -47,6 +49,7 @@ import javax.net.ssl.SSLContext;
|
|||
import javax.net.ssl.SSLSocketFactory;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.Proxy;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
|
@ -172,31 +175,6 @@ public class InvokeHTTPTest {
|
|||
runner.assertNotValid();
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testNotValidWithProxyHostWithoutProxyPort() {
|
||||
runner.setProperty(InvokeHTTP.HTTP_URL, HTTP_LOCALHOST_URL);
|
||||
runner.setProperty(InvokeHTTP.PROXY_HOST, String.class.getSimpleName());
|
||||
runner.assertNotValid();
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testNotValidWithProxyUserWithoutProxyPassword() {
|
||||
runner.setProperty(InvokeHTTP.HTTP_URL, HTTP_LOCALHOST_URL);
|
||||
runner.setProperty(InvokeHTTP.PROXY_USERNAME, String.class.getSimpleName());
|
||||
runner.assertNotValid();
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testNotValidWithProxyUserAndPasswordWithoutProxyHost() {
|
||||
runner.setProperty(InvokeHTTP.HTTP_URL, HTTP_LOCALHOST_URL);
|
||||
runner.setProperty(InvokeHTTP.PROXY_USERNAME, String.class.getSimpleName());
|
||||
runner.setProperty(InvokeHTTP.PROXY_PASSWORD, String.class.getSimpleName());
|
||||
runner.assertNotValid();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNotValidWithPostFormPropertyWithoutFormBodyFormName() {
|
||||
runner.setProperty(InvokeHTTP.HTTP_URL, HTTP_LOCALHOST_URL);
|
||||
|
|
@ -351,39 +329,26 @@ public class InvokeHTTPTest {
|
|||
assertRelationshipStatusCodeEquals(InvokeHTTP.RESPONSE, HTTP_OK);
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testRunGetHttp200SuccessProxyHostPortConfigured() throws InterruptedException {
|
||||
public void testRunGetHttp200SuccessProxyHostPortConfigured() throws InterruptedException, InitializationException {
|
||||
final String mockWebServerUrl = getMockWebServerUrl();
|
||||
final URI uri = URI.create(mockWebServerUrl);
|
||||
|
||||
runner.setProperty(InvokeHTTP.HTTP_URL, mockWebServerUrl);
|
||||
runner.setProperty(InvokeHTTP.PROXY_HOST, uri.getHost());
|
||||
runner.setProperty(InvokeHTTP.PROXY_PORT, Integer.toString(uri.getPort()));
|
||||
final String proxyConfigurationServiceId = ProxyConfigurationService.class.getSimpleName();
|
||||
final ProxyConfigurationService proxyConfigurationService = mock(ProxyConfigurationService.class);
|
||||
when(proxyConfigurationService.getIdentifier()).thenReturn(proxyConfigurationServiceId);
|
||||
runner.addControllerService(proxyConfigurationServiceId, proxyConfigurationService);
|
||||
runner.enableControllerService(proxyConfigurationService);
|
||||
|
||||
mockWebServer.enqueue(new MockResponse().setResponseCode(HTTP_OK));
|
||||
runner.enqueue(FLOW_FILE_CONTENT);
|
||||
runner.run();
|
||||
final ProxyConfiguration proxyConfiguration = new ProxyConfiguration();
|
||||
proxyConfiguration.setProxyType(Proxy.Type.HTTP);
|
||||
proxyConfiguration.setProxyServerHost(uri.getHost());
|
||||
proxyConfiguration.setProxyServerPort(uri.getPort());
|
||||
|
||||
assertRelationshipStatusCodeEquals(InvokeHTTP.RESPONSE, HTTP_OK);
|
||||
final RecordedRequest request = takeRequestCompleted();
|
||||
final String requestLine = request.getRequestLine();
|
||||
|
||||
final String proxyRequestLine = String.format("%s %s HTTP/1.1", HttpMethod.GET.name(), mockWebServerUrl);
|
||||
assertEquals(proxyRequestLine, requestLine);
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testRunGetHttp200SuccessProxyHostPortUserPasswordConfigured() throws InterruptedException {
|
||||
final String mockWebServerUrl = getMockWebServerUrl();
|
||||
final URI uri = URI.create(mockWebServerUrl);
|
||||
when(proxyConfigurationService.getConfiguration()).thenReturn(proxyConfiguration);
|
||||
|
||||
runner.setProperty(InvokeHTTP.HTTP_URL, mockWebServerUrl);
|
||||
runner.setProperty(InvokeHTTP.PROXY_HOST, uri.getHost());
|
||||
runner.setProperty(InvokeHTTP.PROXY_PORT, Integer.toString(uri.getPort()));
|
||||
runner.setProperty(InvokeHTTP.PROXY_USERNAME, String.class.getSimpleName());
|
||||
runner.setProperty(InvokeHTTP.PROXY_PASSWORD, String.class.getName());
|
||||
runner.setProperty(ProxyConfigurationService.PROXY_CONFIGURATION_SERVICE, proxyConfigurationServiceId);
|
||||
|
||||
mockWebServer.enqueue(new MockResponse().setResponseCode(HTTP_OK));
|
||||
runner.enqueue(FLOW_FILE_CONTENT);
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
import org.apache.nifi.util.MockFlowFile;
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class TestBase64EncodeContent {
|
||||
|
||||
@Test
|
||||
public void testRoundTrip() throws IOException {
|
||||
final TestRunner testRunner = TestRunners.newTestRunner(new Base64EncodeContent());
|
||||
|
||||
testRunner.setProperty(Base64EncodeContent.MODE, Base64EncodeContent.ENCODE_MODE);
|
||||
|
||||
testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
|
||||
testRunner.clearTransferState();
|
||||
testRunner.run();
|
||||
|
||||
testRunner.assertAllFlowFilesTransferred(Base64EncodeContent.REL_SUCCESS, 1);
|
||||
|
||||
MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(Base64EncodeContent.REL_SUCCESS).get(0);
|
||||
testRunner.assertQueueEmpty();
|
||||
|
||||
testRunner.setProperty(Base64EncodeContent.MODE, Base64EncodeContent.DECODE_MODE);
|
||||
testRunner.enqueue(flowFile);
|
||||
testRunner.clearTransferState();
|
||||
testRunner.run();
|
||||
testRunner.assertAllFlowFilesTransferred(Base64EncodeContent.REL_SUCCESS, 1);
|
||||
|
||||
flowFile = testRunner.getFlowFilesForRelationship(Base64EncodeContent.REL_SUCCESS).get(0);
|
||||
flowFile.assertContentEquals(new File("src/test/resources/hello.txt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailDecodeNotBase64() throws IOException {
|
||||
final TestRunner testRunner = TestRunners.newTestRunner(new Base64EncodeContent());
|
||||
|
||||
testRunner.setProperty(Base64EncodeContent.MODE, Base64EncodeContent.DECODE_MODE);
|
||||
|
||||
testRunner.enqueue(Paths.get("src/test/resources/hello.txt"));
|
||||
testRunner.clearTransferState();
|
||||
testRunner.run();
|
||||
|
||||
testRunner.assertAllFlowFilesTransferred(Base64EncodeContent.REL_FAILURE, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailDecodeNotBase64ButIsAMultipleOfFourBytes() throws IOException {
|
||||
final TestRunner testRunner = TestRunners.newTestRunner(new Base64EncodeContent());
|
||||
|
||||
testRunner.setProperty(Base64EncodeContent.MODE, Base64EncodeContent.DECODE_MODE);
|
||||
|
||||
testRunner.enqueue("four@@@@multiple".getBytes());
|
||||
testRunner.clearTransferState();
|
||||
testRunner.run();
|
||||
|
||||
testRunner.assertAllFlowFilesTransferred(Base64EncodeContent.REL_FAILURE, 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processors.standard.util.JmsFactory;
|
||||
import org.apache.nifi.processors.standard.util.JmsProperties;
|
||||
import org.apache.nifi.processors.standard.util.WrappedMessageProducer;
|
||||
import org.apache.nifi.util.MockFlowFile;
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.jms.BytesMessage;
|
||||
import javax.jms.Message;
|
||||
import javax.jms.MessageProducer;
|
||||
import javax.jms.Session;
|
||||
import javax.jms.StreamMessage;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public class TestGetJMSQueue {
|
||||
|
||||
@Test
|
||||
public void testSendTextToQueue() throws Exception {
|
||||
PutJMS putJms = new PutJMS();
|
||||
TestRunner putRunner = TestRunners.newTestRunner(putJms);
|
||||
putRunner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
|
||||
putRunner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
|
||||
putRunner.setProperty(JmsProperties.DESTINATION_TYPE, JmsProperties.DESTINATION_TYPE_QUEUE);
|
||||
putRunner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
|
||||
putRunner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
|
||||
|
||||
WrappedMessageProducer wrappedProducer = JmsFactory.createMessageProducer(putRunner.getProcessContext(), true);
|
||||
final Session jmsSession = wrappedProducer.getSession();
|
||||
final MessageProducer producer = wrappedProducer.getProducer();
|
||||
final Message message = jmsSession.createTextMessage("Hello World");
|
||||
|
||||
producer.send(message);
|
||||
jmsSession.commit();
|
||||
|
||||
GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
TestRunner runner = TestRunners.newTestRunner(getJmsQueue);
|
||||
runner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
|
||||
runner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
|
||||
runner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
|
||||
runner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
|
||||
|
||||
runner.run();
|
||||
|
||||
List<MockFlowFile> flowFiles = runner
|
||||
.getFlowFilesForRelationship(new Relationship.Builder().name("success").build());
|
||||
|
||||
assertEquals(1, flowFiles.size());
|
||||
MockFlowFile successFlowFile = flowFiles.get(0);
|
||||
successFlowFile.assertContentEquals("Hello World");
|
||||
successFlowFile.assertAttributeEquals("jms.JMSDestination", "queue.testing");
|
||||
producer.close();
|
||||
jmsSession.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSendBytesToQueue() throws Exception {
|
||||
PutJMS putJms = new PutJMS();
|
||||
TestRunner putRunner = TestRunners.newTestRunner(putJms);
|
||||
putRunner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
|
||||
putRunner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
|
||||
putRunner.setProperty(JmsProperties.DESTINATION_TYPE, JmsProperties.DESTINATION_TYPE_QUEUE);
|
||||
putRunner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
|
||||
putRunner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
|
||||
WrappedMessageProducer wrappedProducer = JmsFactory.createMessageProducer(putRunner.getProcessContext(), true);
|
||||
final Session jmsSession = wrappedProducer.getSession();
|
||||
final MessageProducer producer = wrappedProducer.getProducer();
|
||||
final BytesMessage message = jmsSession.createBytesMessage();
|
||||
message.writeBytes("Hello Bytes".getBytes());
|
||||
|
||||
producer.send(message);
|
||||
jmsSession.commit();
|
||||
|
||||
GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
TestRunner runner = TestRunners.newTestRunner(getJmsQueue);
|
||||
runner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
|
||||
runner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
|
||||
runner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
|
||||
runner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
|
||||
|
||||
runner.run();
|
||||
|
||||
List<MockFlowFile> flowFiles = runner
|
||||
.getFlowFilesForRelationship(new Relationship.Builder().name("success").build());
|
||||
|
||||
assertEquals(1, flowFiles.size());
|
||||
MockFlowFile successFlowFile = flowFiles.get(0);
|
||||
successFlowFile.assertContentEquals("Hello Bytes");
|
||||
successFlowFile.assertAttributeEquals("jms.JMSDestination", "queue.testing");
|
||||
producer.close();
|
||||
jmsSession.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSendStreamToQueue() throws Exception {
|
||||
PutJMS putJms = new PutJMS();
|
||||
TestRunner putRunner = TestRunners.newTestRunner(putJms);
|
||||
putRunner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
|
||||
putRunner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
|
||||
putRunner.setProperty(JmsProperties.DESTINATION_TYPE, JmsProperties.DESTINATION_TYPE_QUEUE);
|
||||
putRunner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
|
||||
putRunner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
|
||||
WrappedMessageProducer wrappedProducer = JmsFactory.createMessageProducer(putRunner.getProcessContext(), true);
|
||||
final Session jmsSession = wrappedProducer.getSession();
|
||||
final MessageProducer producer = wrappedProducer.getProducer();
|
||||
|
||||
final StreamMessage message = jmsSession.createStreamMessage();
|
||||
message.writeBytes("Hello Stream".getBytes());
|
||||
|
||||
producer.send(message);
|
||||
jmsSession.commit();
|
||||
|
||||
GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
TestRunner runner = TestRunners.newTestRunner(getJmsQueue);
|
||||
runner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
|
||||
runner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
|
||||
runner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
|
||||
runner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
|
||||
|
||||
runner.run();
|
||||
|
||||
List<MockFlowFile> flowFiles = runner
|
||||
.getFlowFilesForRelationship(new Relationship.Builder().name("success").build());
|
||||
|
||||
assertEquals(1, flowFiles.size());
|
||||
MockFlowFile successFlowFile = flowFiles.get(0);
|
||||
successFlowFile.assertContentEquals("Hello Stream");
|
||||
successFlowFile.assertAttributeEquals("jms.JMSDestination", "queue.testing");
|
||||
|
||||
producer.close();
|
||||
jmsSession.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import org.apache.nifi.util.MockFlowFile;
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
public class TestHashAttribute {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new HashAttribute());
|
||||
runner.setProperty(HashAttribute.HASH_VALUE_ATTRIBUTE.getName(), "hashValue");
|
||||
runner.setProperty("MDKey1", ".*");
|
||||
runner.setProperty("MDKey2", "(.).*");
|
||||
|
||||
final Map<String, String> attributeMap = new HashMap<>();
|
||||
attributeMap.put("MDKey1", "a");
|
||||
attributeMap.put("MDKey2", "b");
|
||||
runner.enqueue(new byte[0], attributeMap);
|
||||
|
||||
attributeMap.put("MDKey1", "1");
|
||||
attributeMap.put("MDKey2", "2");
|
||||
runner.enqueue(new byte[0], attributeMap);
|
||||
|
||||
attributeMap.put("MDKey1", "a");
|
||||
attributeMap.put("MDKey2", "z");
|
||||
runner.enqueue(new byte[0], attributeMap);
|
||||
|
||||
attributeMap.put("MDKey1", "a");
|
||||
attributeMap.put("MDKey2", "bad");
|
||||
runner.enqueue(new byte[0], attributeMap);
|
||||
|
||||
attributeMap.put("MDKey1", "a");
|
||||
attributeMap.remove("MDKey2");
|
||||
runner.enqueue(new byte[0], attributeMap);
|
||||
|
||||
runner.run(5);
|
||||
|
||||
runner.assertTransferCount(HashAttribute.REL_FAILURE, 1);
|
||||
runner.assertTransferCount(HashAttribute.REL_SUCCESS, 4);
|
||||
|
||||
final List<MockFlowFile> success = runner.getFlowFilesForRelationship(HashAttribute.REL_SUCCESS);
|
||||
final Map<String, Integer> correlationCount = new HashMap<>();
|
||||
for (final MockFlowFile flowFile : success) {
|
||||
final String correlationId = flowFile.getAttribute("hashValue");
|
||||
assertNotNull(correlationId);
|
||||
|
||||
Integer cur = correlationCount.get(correlationId);
|
||||
if (cur == null) {
|
||||
cur = 0;
|
||||
}
|
||||
|
||||
correlationCount.put(correlationId, cur + 1);
|
||||
}
|
||||
|
||||
int twoCount = 0;
|
||||
int oneCount = 0;
|
||||
for (final Integer i : correlationCount.values()) {
|
||||
if (i == 1) {
|
||||
oneCount++;
|
||||
} else if (i == 2) {
|
||||
twoCount++;
|
||||
} else {
|
||||
fail("Got count of " + i);
|
||||
}
|
||||
}
|
||||
|
||||
assertEquals(1, twoCount);
|
||||
assertEquals(2, oneCount);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import org.apache.nifi.util.MockFlowFile;
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public class TestHashContent {
|
||||
|
||||
@Test
|
||||
public void testSHA256() throws IOException {
|
||||
test("SHA-256", "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f");
|
||||
}
|
||||
|
||||
private void test(final String hashAlgorithm, final String expectedHash) throws IOException {
|
||||
final TestRunner runner = TestRunners.newTestRunner(new HashContent());
|
||||
runner.setProperty(org.apache.nifi.processors.standard.HashContent.ATTRIBUTE_NAME, "hash");
|
||||
runner.setProperty(org.apache.nifi.processors.standard.HashContent.HASH_ALGORITHM, hashAlgorithm);
|
||||
|
||||
runner.enqueue(Paths.get("src/test/resources/hello.txt"));
|
||||
|
||||
runner.run();
|
||||
runner.assertQueueEmpty();
|
||||
runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.HashContent.REL_SUCCESS, 1);
|
||||
|
||||
final MockFlowFile outFile = runner.getFlowFilesForRelationship(org.apache.nifi.processors.standard.HashContent.REL_SUCCESS).get(0);
|
||||
final String hashValue = outFile.getAttribute("hash");
|
||||
|
||||
assertEquals(expectedHash, hashValue);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import org.apache.activemq.command.ActiveMQBytesMessage;
|
||||
import org.apache.activemq.command.ActiveMQMapMessage;
|
||||
import org.apache.activemq.command.ActiveMQTextMessage;
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.ProcessSession;
|
||||
import org.apache.nifi.processor.ProcessorInitializationContext;
|
||||
import org.apache.nifi.processor.io.InputStreamCallback;
|
||||
import org.apache.nifi.processors.standard.util.JmsProcessingSummary;
|
||||
import org.apache.nifi.stream.io.StreamUtils;
|
||||
import org.apache.nifi.util.MockProcessContext;
|
||||
import org.apache.nifi.util.MockProcessorInitializationContext;
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.jms.BytesMessage;
|
||||
import javax.jms.JMSException;
|
||||
import javax.jms.MapMessage;
|
||||
import javax.jms.TextMessage;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public class TestJmsConsumer {
|
||||
|
||||
static protected MapMessage createMapMessage() throws JMSException {
|
||||
MapMessage mapMessage = new ActiveMQMapMessage();
|
||||
mapMessage.setString("name", "Arnold");
|
||||
mapMessage.setInt("age", 97);
|
||||
mapMessage.setDouble("xyz", 89686.564);
|
||||
mapMessage.setBoolean("good", true);
|
||||
return mapMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test method for {@link org.apache.nifi.processors.standard.JmsConsumer#createMapMessageAttrs(javax.jms.MapMessage)}.
|
||||
*
|
||||
* @throws JMSException jms
|
||||
*/
|
||||
@Test
|
||||
public void testCreateMapMessageValues() throws JMSException {
|
||||
|
||||
MapMessage mapMessage = createMapMessage();
|
||||
|
||||
Map<String, String> mapMessageValues = JmsConsumer.createMapMessageValues(mapMessage);
|
||||
assertEquals(4, mapMessageValues.size());
|
||||
assertEquals("Arnold", mapMessageValues.get(JmsConsumer.MAP_MESSAGE_PREFIX + "name"));
|
||||
assertEquals("97", mapMessageValues.get(JmsConsumer.MAP_MESSAGE_PREFIX + "age"));
|
||||
assertEquals("89686.564", mapMessageValues.get(JmsConsumer.MAP_MESSAGE_PREFIX + "xyz"));
|
||||
assertEquals("true", mapMessageValues.get(JmsConsumer.MAP_MESSAGE_PREFIX + "good"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test MapMessage to FlowFile conversion
|
||||
*
|
||||
* @throws java.lang.Exception ex
|
||||
*/
|
||||
@Test
|
||||
public void testMap2FlowFileMapMessage() throws Exception {
|
||||
|
||||
TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
|
||||
MapMessage mapMessage = createMapMessage();
|
||||
|
||||
ProcessContext context = runner.getProcessContext();
|
||||
ProcessSession session = runner.getProcessSessionFactory().createSession();
|
||||
ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());
|
||||
|
||||
JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, mapMessage, true, pic.getLogger());
|
||||
|
||||
assertEquals(0, summary.getBytesReceived(), "MapMessage should not create FlowFile content");
|
||||
|
||||
Map<String, String> attributes = summary.getLastFlowFile().getAttributes();
|
||||
assertEquals("Arnold", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "name"));
|
||||
assertEquals("97", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "age"));
|
||||
assertEquals("89686.564", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "xyz"));
|
||||
assertEquals("true", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "good"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMap2FlowFileTextMessage() throws Exception {
|
||||
|
||||
TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
|
||||
TextMessage textMessage = new ActiveMQTextMessage();
|
||||
|
||||
String payload = "Hello world!";
|
||||
textMessage.setText(payload);
|
||||
|
||||
ProcessContext context = runner.getProcessContext();
|
||||
ProcessSession session = runner.getProcessSessionFactory().createSession();
|
||||
ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());
|
||||
|
||||
JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, textMessage, true, pic.getLogger());
|
||||
|
||||
assertEquals(payload.length(), summary.getLastFlowFile().getSize(), "TextMessage content length should equal to FlowFile content size");
|
||||
|
||||
final byte[] buffer = new byte[payload.length()];
|
||||
runner.clearTransferState();
|
||||
|
||||
session.read(summary.getLastFlowFile(), new InputStreamCallback() {
|
||||
@Override
|
||||
public void process(InputStream in) throws IOException {
|
||||
StreamUtils.fillBuffer(in, buffer, false);
|
||||
}
|
||||
});
|
||||
|
||||
String contentString = new String(buffer, "UTF-8");
|
||||
assertEquals(payload, contentString);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test BytesMessage to FlowFile conversion
|
||||
*
|
||||
* @throws java.lang.Exception ex
|
||||
*/
|
||||
@Test
|
||||
public void testMap2FlowFileBytesMessage() throws Exception {
|
||||
|
||||
TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
|
||||
BytesMessage bytesMessage = new ActiveMQBytesMessage();
|
||||
|
||||
String sourceString = "Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data.!";
|
||||
byte[] payload = sourceString.getBytes("UTF-8");
|
||||
bytesMessage.writeBytes(payload);
|
||||
bytesMessage.reset();
|
||||
|
||||
ProcessContext context = runner.getProcessContext();
|
||||
ProcessSession session = runner.getProcessSessionFactory().createSession();
|
||||
ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());
|
||||
|
||||
JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, bytesMessage, true, pic.getLogger());
|
||||
|
||||
assertEquals(payload.length, summary.getLastFlowFile().getSize(), "BytesMessage content length should equal to FlowFile content size");
|
||||
|
||||
final byte[] buffer = new byte[payload.length];
|
||||
runner.clearTransferState();
|
||||
|
||||
session.read(summary.getLastFlowFile(), new InputStreamCallback() {
|
||||
@Override
|
||||
public void process(InputStream in) throws IOException {
|
||||
StreamUtils.fillBuffer(in, buffer, false);
|
||||
}
|
||||
});
|
||||
|
||||
String contentString = new String(buffer, "UTF-8");
|
||||
assertEquals(sourceString, contentString);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,615 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.nifi.processors.standard;
|
||||
|
||||
import org.apache.nifi.processor.ProcessContext;
|
||||
import org.apache.nifi.processor.Relationship;
|
||||
import org.apache.nifi.processors.standard.util.JmsFactory;
|
||||
import org.apache.nifi.processors.standard.util.JmsProperties;
|
||||
import org.apache.nifi.processors.standard.util.WrappedMessageProducer;
|
||||
import org.apache.nifi.provenance.ProvenanceEventRecord;
|
||||
import org.apache.nifi.util.MockFlowFile;
|
||||
import org.apache.nifi.util.TestRunner;
|
||||
import org.apache.nifi.util.TestRunners;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import javax.jms.Connection;
|
||||
import javax.jms.JMSException;
|
||||
import javax.jms.Message;
|
||||
import javax.jms.MessageProducer;
|
||||
import javax.jms.Session;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Queue;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.spy;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Disabled("Processor requires updates to work with Mockito 2.x, but is deprecated.")
|
||||
public class TestPutJMS {
|
||||
|
||||
private final String TEST_PROVIDER = JmsProperties.ACTIVEMQ_PROVIDER;
|
||||
private final String TEST_URL = "vm://localhost?broker.persistent=false";
|
||||
private final String TEST_DEST_TYPE = JmsProperties.DESTINATION_TYPE_QUEUE;
|
||||
private final String TEST_DEST_NAME = "queue.testing";
|
||||
private final String TEST_ACK_MODE = JmsProperties.ACK_MODE_AUTO;
|
||||
|
||||
private String testQueueSuffix() {
|
||||
final StackTraceElement[] trace = Thread.currentThread().getStackTrace();
|
||||
return "." + trace[2].getMethodName();
|
||||
}
|
||||
|
||||
private void injectFieldValue(Class klass, Object instance, String fieldName, Object fieldValue) throws NoSuchFieldException, IllegalAccessException {
|
||||
Field field = klass.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(instance, fieldValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetRelationships() {
|
||||
final PutJMS putJMS = new PutJMS();
|
||||
final Set<Relationship> relationships = putJMS.getRelationships();
|
||||
assertEquals(2, relationships.size());
|
||||
assertTrue(relationships.contains(PutJMS.REL_FAILURE));
|
||||
assertTrue(relationships.contains(PutJMS.REL_SUCCESS));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCleanupResources() throws JMSException, NoSuchFieldException, IllegalAccessException {
|
||||
final PutJMS putJMS = new PutJMS();
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
|
||||
final Queue<WrappedMessageProducer> wrappedMessageProducerQueue = spy(new LinkedBlockingQueue<>());
|
||||
injectFieldValue(PutJMS.class, putJMS, "producerQueue", wrappedMessageProducerQueue);
|
||||
|
||||
final WrappedMessageProducer wrappedProducer = JmsFactory.createMessageProducer(runnerPut.getProcessContext(), true);
|
||||
wrappedMessageProducerQueue.offer(wrappedProducer);
|
||||
|
||||
assertNotNull(wrappedMessageProducerQueue.peek());
|
||||
putJMS.cleanupResources();
|
||||
assertNull(wrappedMessageProducerQueue.peek());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateMessageDirectly() throws JMSException {
|
||||
final PutJMS putJMS = new PutJMS();
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
|
||||
|
||||
final WrappedMessageProducer wrappedProducer = JmsFactory.createMessageProducer(runnerPut.getProcessContext(), true);
|
||||
final Session jmsSession = wrappedProducer.getSession();
|
||||
final MessageProducer producer = wrappedProducer.getProducer();
|
||||
final Message message = jmsSession.createTextMessage("createMessageDirectly");
|
||||
|
||||
producer.send(message);
|
||||
jmsSession.commit();
|
||||
|
||||
final GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
|
||||
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerGet.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> flowFiles = runnerGet.getFlowFilesForRelationship(
|
||||
new Relationship.Builder().name("success").build());
|
||||
|
||||
assertEquals(1, flowFiles.size());
|
||||
final MockFlowFile successFlowFile = flowFiles.get(0);
|
||||
successFlowFile.assertContentEquals("createMessageDirectly");
|
||||
successFlowFile.assertAttributeEquals("jms.JMSDestination", TEST_DEST_NAME + testQueueSuffix());
|
||||
producer.close();
|
||||
jmsSession.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutGetAttributesAndProps() {
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.REPLY_TO_QUEUE, TEST_DEST_NAME + testQueueSuffix() + ".reply");
|
||||
runnerPut.setProperty(JmsProperties.ATTRIBUTES_TO_JMS_PROPS, "true");
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
attributes.put("jms.string", "banana");
|
||||
attributes.put("jms.integer", "50");
|
||||
attributes.put("jms.integer.type", "integer");
|
||||
attributes.put("jms.float", "3.14159");
|
||||
attributes.put("jms.float.type", "float");
|
||||
attributes.put("jms.boolean", "true");
|
||||
attributes.put("jms.boolean.type", "boolean");
|
||||
attributes.put("jms.long", "123456789");
|
||||
attributes.put("jms.long.type", "long");
|
||||
attributes.put("jms.short", "16384");
|
||||
attributes.put("jms.short.type", "short");
|
||||
attributes.put("jms.byte", "127");
|
||||
attributes.put("jms.byte.type", "byte");
|
||||
attributes.put("jms.double", "3.1415626547");
|
||||
attributes.put("jms.double.type", "double");
|
||||
attributes.put("jms.object", "{\"id\":215, \"name\": \"john doe\"}");
|
||||
attributes.put("jms.object.type", "object");
|
||||
attributes.put("jms.eyes", "blue");
|
||||
attributes.put("jms.eyes.type", "color");
|
||||
attributes.put("jms.badinteger", "3.14");
|
||||
attributes.put("jms.badinteger.type", "integer");
|
||||
runnerPut.enqueue("putGetMessage".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
final GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerGet.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROPS_TO_ATTRIBUTES, "true");
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
assertEquals(1, runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS).size());
|
||||
|
||||
final List<MockFlowFile> flowFilesGet = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
|
||||
assertEquals(1, flowFilesGet.size());
|
||||
final MockFlowFile successFlowFile = flowFilesGet.get(0);
|
||||
|
||||
successFlowFile.assertContentEquals("putGetMessage");
|
||||
successFlowFile.assertAttributeEquals("jms.JMSDestination", TEST_DEST_NAME + testQueueSuffix());
|
||||
successFlowFile.assertAttributeEquals("jms.JMSReplyTo", "queue://" + TEST_DEST_NAME + testQueueSuffix() + ".reply");
|
||||
successFlowFile.assertAttributeEquals("jms.string", "banana");
|
||||
successFlowFile.assertAttributeEquals("jms.integer", "50");
|
||||
successFlowFile.assertAttributeEquals("jms.float", "3.14159");
|
||||
successFlowFile.assertAttributeEquals("jms.boolean", "true");
|
||||
successFlowFile.assertAttributeEquals("jms.long", "123456789");
|
||||
successFlowFile.assertAttributeEquals("jms.short", "16384");
|
||||
successFlowFile.assertAttributeEquals("jms.byte", "127");
|
||||
successFlowFile.assertAttributeEquals("jms.double", "3.1415626547");
|
||||
successFlowFile.assertAttributeEquals("jms.object", "{\"id\":215, \"name\": \"john doe\"}");
|
||||
successFlowFile.assertAttributeEquals("jms.eyes", null);
|
||||
successFlowFile.assertAttributeEquals("jms.badinteger", null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutGetMessageTypes() {
|
||||
final GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerGet.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROPS_TO_ATTRIBUTES, "true");
|
||||
|
||||
//------------------------------------------------------------
|
||||
final PutJMS putJMStext = spy(new PutJMS());
|
||||
final TestRunner runnerPutText = TestRunners.newTestRunner(putJMStext);
|
||||
runnerPutText.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPutText.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPutText.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPutText.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPutText.setProperty(JmsProperties.MESSAGE_TYPE, JmsProperties.MSG_TYPE_TEXT);
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
runnerPutText.enqueue("putGetTextMessage", attributes);
|
||||
|
||||
runnerPutText.run();
|
||||
|
||||
assertEquals(0, runnerPutText.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPutText.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> ffText = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(1, ffText.size());
|
||||
final MockFlowFile successText = ffText.get(0);
|
||||
successText.assertContentEquals("putGetTextMessage");
|
||||
|
||||
//------------------------------------------------------------
|
||||
final PutJMS putJMSempty = spy(new PutJMS());
|
||||
final TestRunner runnerPutEmpty = TestRunners.newTestRunner(putJMSempty);
|
||||
runnerPutEmpty.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPutEmpty.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPutEmpty.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPutEmpty.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPutEmpty.setProperty(JmsProperties.MESSAGE_TYPE, JmsProperties.MSG_TYPE_EMPTY);
|
||||
|
||||
final Map<String, String> attributesEmpty = new HashMap<>();
|
||||
attributesEmpty.put("filename", "file1.txt");
|
||||
runnerPutEmpty.enqueue("putGetEmptyMessage", attributesEmpty);
|
||||
|
||||
runnerPutEmpty.run();
|
||||
|
||||
assertEquals(0, runnerPutEmpty.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPutEmpty.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
runnerGet.clearTransferState();
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> ffEmpty = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(1, ffEmpty.size());
|
||||
final MockFlowFile successEmpty = ffEmpty.get(0);
|
||||
successEmpty.assertContentEquals("");
|
||||
|
||||
//------------------------------------------------------------
|
||||
final PutJMS putJMSstream = spy(new PutJMS());
|
||||
final TestRunner runnerPutStream = TestRunners.newTestRunner(putJMSstream);
|
||||
runnerPutStream.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPutStream.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPutStream.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPutStream.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPutStream.setProperty(JmsProperties.MESSAGE_TYPE, JmsProperties.MSG_TYPE_STREAM);
|
||||
|
||||
final Map<String, String> attributesStream = new HashMap<>();
|
||||
attributesStream.put("filename", "file1.txt");
|
||||
runnerPutStream.enqueue("putGetStreamMessage", attributesStream);
|
||||
|
||||
runnerGet.clearTransferState();
|
||||
runnerPutStream.run();
|
||||
|
||||
assertEquals(0, runnerPutStream.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPutStream.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> ffStream = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(1, ffStream.size());
|
||||
final MockFlowFile successStream = ffStream.get(0);
|
||||
successStream.assertContentEquals("putGetStreamMessage");
|
||||
|
||||
//------------------------------------------------------------
|
||||
final PutJMS putJMSmap = spy(new PutJMS());
|
||||
final TestRunner runnerPutMap = TestRunners.newTestRunner(putJMSmap);
|
||||
runnerPutMap.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPutMap.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPutMap.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPutMap.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPutMap.setProperty(JmsProperties.MESSAGE_TYPE, JmsProperties.MSG_TYPE_MAP);
|
||||
|
||||
final Map<String, String> attributesMap = new HashMap<>();
|
||||
attributesMap.put("filename", "file1.txt");
|
||||
runnerPutMap.enqueue("putGetMapMessage", attributesMap);
|
||||
|
||||
runnerGet.clearTransferState();
|
||||
runnerPutMap.run();
|
||||
|
||||
assertEquals(0, runnerPutMap.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPutMap.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> ffMap = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(1, ffMap.size());
|
||||
final MockFlowFile successMap = ffMap.get(0);
|
||||
successMap.assertContentEquals("");
|
||||
|
||||
//------------------------------------------------------------
|
||||
final PutJMS putJMSByte = spy(new PutJMS());
|
||||
final TestRunner runnerPutByte = TestRunners.newTestRunner(putJMSByte);
|
||||
runnerPutByte.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPutByte.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPutByte.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPutByte.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPutByte.setProperty(JmsProperties.MESSAGE_TYPE, JmsProperties.MSG_TYPE_BYTE);
|
||||
|
||||
final Map<String, String> attributesByte = new HashMap<>();
|
||||
attributesByte.put("filename", "file1.txt");
|
||||
runnerPutByte.enqueue("putGetTextMessage", attributesByte);
|
||||
|
||||
runnerPutByte.run();
|
||||
|
||||
assertEquals(0, runnerPutByte.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPutByte.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
runnerGet.clearTransferState();
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> ffByte = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(1, ffByte.size());
|
||||
final MockFlowFile successByte = ffByte.get(0);
|
||||
successByte.assertContentEquals("putGetTextMessage");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTTL() throws InterruptedException {
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.MESSAGE_TTL, "3 s");
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
runnerPut.enqueue("ttl10secNotExpired".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
final GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerGet.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
|
||||
runnerGet.setProperty(JmsProperties.TIMEOUT, "1 s");
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> flowFiles1 = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(1, flowFiles1.size());
|
||||
final MockFlowFile successFlowFile1 = flowFiles1.get(0);
|
||||
successFlowFile1.assertContentEquals("ttl10secNotExpired");
|
||||
|
||||
runnerPut.clearTransferState();
|
||||
runnerGet.clearTransferState();
|
||||
|
||||
runnerPut.setProperty(JmsProperties.MESSAGE_TTL, "1 s");
|
||||
runnerPut.enqueue("ttl1secExpired".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
Thread.sleep(2000L);
|
||||
runnerGet.run();
|
||||
|
||||
final List<MockFlowFile> flowFiles2 = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
assertEquals(0, flowFiles2.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailureOnFileExceedsBufferSize() {
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.MAX_BUFFER_SIZE, "10 B");
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
runnerPut.enqueue("failureOnFileExceedsBufferSize".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBadMessagePriorityValueFails() {
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.MESSAGE_PRIORITY, "negative one");
|
||||
assertThrows(NumberFormatException.class, () -> {
|
||||
assertEquals(PutJMS.DEFAULT_MESSAGE_PRIORITY, runnerPut.getProcessContext().getProperty(JmsProperties.MESSAGE_PRIORITY).asInteger().intValue());
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBadMessagePriorityRunSucceeds() {
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.MESSAGE_PRIORITY, "negative one");
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
runnerPut.enqueue("badMessagePriorityRunSucceeds".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutSendRoutesToFailure() throws JMSException, NoSuchFieldException, IllegalAccessException {
|
||||
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
|
||||
final ProcessContext context = runnerPut.getProcessContext();
|
||||
|
||||
final Queue<WrappedMessageProducer> wrappedMessageProducerQueue = spy(new LinkedBlockingQueue<>());
|
||||
injectFieldValue(PutJMS.class, putJMS, "producerQueue", wrappedMessageProducerQueue);
|
||||
|
||||
final WrappedMessageProducer wrappedMessageProducer = spy(JmsFactory.createMessageProducer(context, true));
|
||||
final MessageProducer messageProducer = spy(wrappedMessageProducer.getProducer());
|
||||
|
||||
doAnswer(new Answer<WrappedMessageProducer>() {
|
||||
@Override
|
||||
public WrappedMessageProducer answer(InvocationOnMock invocationOnMock) {
|
||||
return wrappedMessageProducer;
|
||||
}
|
||||
}).when(wrappedMessageProducerQueue).poll();
|
||||
assertEquals(wrappedMessageProducer, wrappedMessageProducerQueue.poll());
|
||||
|
||||
doAnswer(new Answer<MessageProducer>() {
|
||||
@Override
|
||||
public MessageProducer answer(InvocationOnMock invocationOnMock) {
|
||||
return messageProducer;
|
||||
}
|
||||
}).when(wrappedMessageProducer).getProducer();
|
||||
|
||||
doThrow(new JMSException("force send to fail")).when(messageProducer).send(any(Message.class));
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
runnerPut.enqueue("putSendRoutesToFailure".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
|
||||
final List<MockFlowFile> flowFilesFail = runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE);
|
||||
assertEquals(1, flowFilesFail.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutCommitRoutesToFailure() throws JMSException, NoSuchFieldException, IllegalAccessException {
|
||||
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
|
||||
final ProcessContext context = runnerPut.getProcessContext();
|
||||
final Queue<WrappedMessageProducer> wrappedMessageProducerQueue = spy(new LinkedBlockingQueue<>());
|
||||
injectFieldValue(PutJMS.class, putJMS, "producerQueue", wrappedMessageProducerQueue);
|
||||
|
||||
final WrappedMessageProducer wrappedMessageProducer = spy(JmsFactory.createMessageProducer(context, true));
|
||||
final MessageProducer messageProducer = spy(wrappedMessageProducer.getProducer());
|
||||
final Connection connection = JmsFactory.createConnection(context);
|
||||
final Session jmsSession = spy(JmsFactory.createSession(context, connection, true));
|
||||
|
||||
doAnswer(new Answer<WrappedMessageProducer>() {
|
||||
@Override
|
||||
public WrappedMessageProducer answer(InvocationOnMock invocationOnMock) {
|
||||
return wrappedMessageProducer;
|
||||
}
|
||||
}).when(wrappedMessageProducerQueue).poll();
|
||||
|
||||
doAnswer(new Answer<MessageProducer>() {
|
||||
@Override
|
||||
public MessageProducer answer(InvocationOnMock invocationOnMock) {
|
||||
return messageProducer;
|
||||
}
|
||||
}).when(wrappedMessageProducer).getProducer();
|
||||
|
||||
doAnswer(new Answer<Session>() {
|
||||
@Override
|
||||
public Session answer(InvocationOnMock invocationOnMock) {
|
||||
return jmsSession;
|
||||
}
|
||||
}).when(wrappedMessageProducer).getSession();
|
||||
|
||||
doThrow(new JMSException("force commit to fail")).when(jmsSession).commit();
|
||||
|
||||
final Map<String, String> attributes = new HashMap<>();
|
||||
attributes.put("filename", "file1.txt");
|
||||
runnerPut.enqueue("putCommitRoutesToFailure".getBytes(), attributes);
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
|
||||
final List<MockFlowFile> flowFilesFail = runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE);
|
||||
assertEquals(1, flowFilesFail.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPutProvenanceSendEventTransitUri() {
|
||||
final PutJMS putJMS = spy(new PutJMS());
|
||||
final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
|
||||
runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerPut.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
|
||||
runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerPut.setProperty(JmsProperties.ATTRIBUTES_TO_JMS_PROPS, "true");
|
||||
|
||||
runnerPut.enqueue("putGetMessage".getBytes());
|
||||
|
||||
runnerPut.run();
|
||||
|
||||
assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
|
||||
assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
|
||||
|
||||
final List<ProvenanceEventRecord> putProvenanceEvents = runnerPut.getProvenanceEvents();
|
||||
|
||||
assertEquals(1, putProvenanceEvents.size());
|
||||
// Verify the transitUri is the same as that configured in the properties
|
||||
assertEquals(TEST_URL, putProvenanceEvents.get(0).getTransitUri());
|
||||
|
||||
final GetJMSQueue getJmsQueue = new GetJMSQueue();
|
||||
final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
|
||||
runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
|
||||
runnerGet.setProperty(JmsProperties.URL, TEST_URL);
|
||||
runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
|
||||
runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
|
||||
|
||||
runnerGet.run();
|
||||
|
||||
assertEquals(1, runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS).size());
|
||||
|
||||
final List<MockFlowFile> flowFilesGet = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
|
||||
|
||||
assertEquals(1, flowFilesGet.size());
|
||||
final MockFlowFile successFlowFile = flowFilesGet.get(0);
|
||||
|
||||
successFlowFile.assertContentEquals("putGetMessage");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package TestGetHTTP
|
||||
|
||||
import javax.servlet.http.HttpServletResponse
|
||||
|
||||
response.setStatus(HttpServletResponse.SC_OK)
|
||||
return
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package TestGetHTTP
|
||||
|
||||
import javax.servlet.http.HttpServletResponse
|
||||
|
||||
final string = request.parameterMap['string']
|
||||
if (!string || string.size() < 1){
|
||||
response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
|
||||
return
|
||||
}
|
||||
print URLDecoder.decode(string[0] as String, 'UTF-8').reverse()
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package TestPostHTTP
|
||||
|
||||
import javax.servlet.http.HttpServletResponse
|
||||
|
||||
final contents = request.reader.text
|
||||
if (!contents || contents.size() < 1){
|
||||
response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
|
||||
return
|
||||
}
|
||||
print "Thank you for submitting flowfile content ${contents}"
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package TestPostHTTP
|
||||
|
||||
import javax.servlet.http.HttpServletResponse
|
||||
|
||||
final string = request.parameterMap['string']
|
||||
if (!string || string.size() < 1){
|
||||
response.setStatus(HttpServletResponse.SC_BAD_REQUEST)
|
||||
return
|
||||
}
|
||||
print URLDecoder.decode(string[0] as String, 'UTF-8').reverse()
|
||||
Loading…
Reference in New Issue