NIFI-10161 Added Gzip Content-Encoding to InvokeHTTP and ListenHTTP

Signed-off-by: Pierre Villard <pierre.villard.fr@gmail.com>

This closes #6150.
This commit is contained in:
exceptionfactory 2022-06-23 11:51:17 -05:00 committed by Pierre Villard
parent b6a32a9c5d
commit d8ebfb25d0
No known key found for this signature in database
GPG Key ID: F92A93B30C07C6D5
5 changed files with 199 additions and 47 deletions

View File

@ -73,6 +73,9 @@ import okhttp3.RequestBody;
import okhttp3.Response; import okhttp3.Response;
import okhttp3.ResponseBody; import okhttp3.ResponseBody;
import okio.BufferedSink; import okio.BufferedSink;
import okio.GzipSink;
import okio.Okio;
import okio.Source;
import org.apache.commons.io.input.TeeInputStream; import org.apache.commons.io.input.TeeInputStream;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperties; import org.apache.nifi.annotation.behavior.DynamicProperties;
@ -103,6 +106,7 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.standard.http.ContentEncodingStrategy;
import org.apache.nifi.processors.standard.http.FlowFileNamingStrategy; import org.apache.nifi.processors.standard.http.FlowFileNamingStrategy;
import org.apache.nifi.processors.standard.http.CookieStrategy; import org.apache.nifi.processors.standard.http.CookieStrategy;
import org.apache.nifi.processors.standard.util.ProxyAuthenticator; import org.apache.nifi.processors.standard.util.ProxyAuthenticator;
@ -186,6 +190,7 @@ public class InvokeHTTP extends AbstractProcessor {
private static final Pattern DYNAMIC_FORM_PARAMETER_NAME = Pattern.compile("post:form:(?<formDataName>.*)$"); private static final Pattern DYNAMIC_FORM_PARAMETER_NAME = Pattern.compile("post:form:(?<formDataName>.*)$");
private static final String FORM_DATA_NAME_GROUP = "formDataName"; private static final String FORM_DATA_NAME_GROUP = "formDataName";
private static final String CONTENT_ENCODING_HEADER = "Content-Encoding";
// properties // properties
public static final PropertyDescriptor PROP_METHOD = new PropertyDescriptor.Builder() public static final PropertyDescriptor PROP_METHOD = new PropertyDescriptor.Builder()
@ -329,6 +334,15 @@ public class InvokeHTTP extends AbstractProcessor {
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build(); .build();
public static final PropertyDescriptor PROP_CONTENT_ENCODING = new PropertyDescriptor.Builder()
.name("Content-Encoding")
.displayName("Content-Encoding")
.description("HTTP Content-Encoding applied to request body during transmission. The receiving server must support the selected encoding to avoid request failures.")
.required(true)
.defaultValue(ContentEncodingStrategy.DISABLED.getValue())
.allowableValues(ContentEncodingStrategy.class)
.build();
public static final PropertyDescriptor PROP_CONTENT_TYPE = new PropertyDescriptor.Builder() public static final PropertyDescriptor PROP_CONTENT_TYPE = new PropertyDescriptor.Builder()
.name("Content-Type") .name("Content-Type")
.description("The Content-Type to specify for when content is being transmitted through a PUT, POST or PATCH. " .description("The Content-Type to specify for when content is being transmitted through a PUT, POST or PATCH. "
@ -564,6 +578,7 @@ public class InvokeHTTP extends AbstractProcessor {
PROP_DIGEST_AUTH, PROP_DIGEST_AUTH,
PROP_OUTPUT_RESPONSE_REGARDLESS, PROP_OUTPUT_RESPONSE_REGARDLESS,
PROP_ADD_HEADERS_TO_REQUEST, PROP_ADD_HEADERS_TO_REQUEST,
PROP_CONTENT_ENCODING,
PROP_CONTENT_TYPE, PROP_CONTENT_TYPE,
PROP_SEND_BODY, PROP_SEND_BODY,
PROP_USE_CHUNKED_ENCODING, PROP_USE_CHUNKED_ENCODING,
@ -1112,6 +1127,12 @@ public class InvokeHTTP extends AbstractProcessor {
} }
} }
final String contentEncoding = context.getProperty(PROP_CONTENT_ENCODING).getValue();
final ContentEncodingStrategy contentEncodingStrategy = ContentEncodingStrategy.valueOf(contentEncoding);
if (ContentEncodingStrategy.GZIP == contentEncodingStrategy) {
requestBuilder.addHeader(CONTENT_ENCODING_HEADER, ContentEncodingStrategy.GZIP.getValue().toLowerCase());
}
// set the request method // set the request method
String method = trimToEmpty(context.getProperty(PROP_METHOD).evaluateAttributeExpressions(requestFlowFile).getValue()).toUpperCase(); String method = trimToEmpty(context.getProperty(PROP_METHOD).evaluateAttributeExpressions(requestFlowFile).getValue()).toUpperCase();
switch (method) { switch (method) {
@ -1119,15 +1140,15 @@ public class InvokeHTTP extends AbstractProcessor {
requestBuilder.get(); requestBuilder.get();
break; break;
case POST_METHOD: case POST_METHOD:
RequestBody requestBody = getRequestBodyToSend(session, context, requestFlowFile); RequestBody requestBody = getRequestBodyToSend(session, context, requestFlowFile, contentEncodingStrategy);
requestBuilder.post(requestBody); requestBuilder.post(requestBody);
break; break;
case PUT_METHOD: case PUT_METHOD:
requestBody = getRequestBodyToSend(session, context, requestFlowFile); requestBody = getRequestBodyToSend(session, context, requestFlowFile, contentEncodingStrategy);
requestBuilder.put(requestBody); requestBuilder.put(requestBody);
break; break;
case PATCH_METHOD: case PATCH_METHOD:
requestBody = getRequestBodyToSend(session, context, requestFlowFile); requestBody = getRequestBodyToSend(session, context, requestFlowFile, contentEncodingStrategy);
requestBuilder.patch(requestBody); requestBuilder.patch(requestBody);
break; break;
case HEAD_METHOD: case HEAD_METHOD:
@ -1149,8 +1170,9 @@ public class InvokeHTTP extends AbstractProcessor {
} }
private RequestBody getRequestBodyToSend(final ProcessSession session, final ProcessContext context, private RequestBody getRequestBodyToSend(final ProcessSession session, final ProcessContext context,
final FlowFile requestFlowFile) { final FlowFile requestFlowFile,
final ContentEncodingStrategy contentEncodingStrategy
) {
boolean sendBody = context.getProperty(PROP_SEND_BODY).asBoolean(); boolean sendBody = context.getProperty(PROP_SEND_BODY).asBoolean();
String evalContentType = context.getProperty(PROP_CONTENT_TYPE) String evalContentType = context.getProperty(PROP_CONTENT_TYPE)
@ -1168,6 +1190,7 @@ public class InvokeHTTP extends AbstractProcessor {
} }
} }
final boolean contentLengthUnknown = useChunked || ContentEncodingStrategy.GZIP == contentEncodingStrategy;
RequestBody requestBody = new RequestBody() { RequestBody requestBody = new RequestBody() {
@Nullable @Nullable
@Override @Override
@ -1176,13 +1199,25 @@ public class InvokeHTTP extends AbstractProcessor {
} }
@Override @Override
public void writeTo(BufferedSink sink) { public void writeTo(final BufferedSink sink) throws IOException {
session.exportTo(requestFlowFile, sink.outputStream()); final BufferedSink outputSink = (ContentEncodingStrategy.GZIP == contentEncodingStrategy)
? Okio.buffer(new GzipSink(sink))
: sink;
session.read(requestFlowFile, inputStream -> {
final Source source = Okio.source(inputStream);
outputSink.writeAll(source);
});
// Close Output Sink for gzip to write trailing bytes
if (ContentEncodingStrategy.GZIP == contentEncodingStrategy) {
outputSink.close();
}
} }
@Override @Override
public long contentLength() { public long contentLength() {
return useChunked ? -1 : requestFlowFile.getSize(); return contentLengthUnknown ? -1 : requestFlowFile.getSize();
} }
}; };

View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.standard.http;
import org.apache.nifi.components.DescribedValue;
/**
* HTTP Content-Encoding configuration strategy
*/
public enum ContentEncodingStrategy implements DescribedValue {
DISABLED("Content encoding not applied during transmission"),
GZIP( "Gzip content encoding and HTTP Content-Encoding header applied during transmission");
private final String description;
ContentEncodingStrategy(final String description) {
this.description = description;
}
@Override
public String getValue() {
return name();
}
@Override
public String getDisplayName() {
return name();
}
@Override
public String getDescription() {
return description;
}
}

View File

@ -101,6 +101,7 @@ public class ListenHTTPServlet extends HttpServlet {
public static final String GZIPPED_HEADER = "flowfile-gzipped"; public static final String GZIPPED_HEADER = "flowfile-gzipped";
public static final String PROTOCOL_VERSION_HEADER = "x-nifi-transfer-protocol-version"; public static final String PROTOCOL_VERSION_HEADER = "x-nifi-transfer-protocol-version";
public static final String PROTOCOL_VERSION = "3"; public static final String PROTOCOL_VERSION = "3";
protected static final String CONTENT_ENCODING_HEADER = "Content-Encoding";
private final AtomicLong filesReceived = new AtomicLong(0L); private final AtomicLong filesReceived = new AtomicLong(0L);
private final AtomicBoolean spaceAvailable = new AtomicBoolean(true); private final AtomicBoolean spaceAvailable = new AtomicBoolean(true);
@ -193,7 +194,10 @@ public class ListenHTTPServlet extends HttpServlet {
} }
response.setHeader("Content-Type", MediaType.TEXT_PLAIN); response.setHeader("Content-Type", MediaType.TEXT_PLAIN);
final boolean contentGzipped = Boolean.parseBoolean(request.getHeader(GZIPPED_HEADER)); final boolean flowFileGzipped = Boolean.parseBoolean(request.getHeader(GZIPPED_HEADER));
final String contentEncoding = request.getHeader(CONTENT_ENCODING_HEADER);
final boolean contentEncodingGzip = ACCEPT_ENCODING_VALUE.equals(contentEncoding);
final boolean contentGzipped = flowFileGzipped || contentEncodingGzip;
final X509Certificate[] certs = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate"); final X509Certificate[] certs = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
foundSubject = DEFAULT_FOUND_SUBJECT; foundSubject = DEFAULT_FOUND_SUBJECT;

View File

@ -19,10 +19,13 @@ package org.apache.nifi.processors.standard;
import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest; import okhttp3.mockwebserver.RecordedRequest;
import okio.Buffer;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.oauth2.OAuth2AccessTokenProvider; import org.apache.nifi.oauth2.OAuth2AccessTokenProvider;
import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processors.standard.http.ContentEncodingStrategy;
import org.apache.nifi.processors.standard.http.FlowFileNamingStrategy; import org.apache.nifi.processors.standard.http.FlowFileNamingStrategy;
import org.apache.nifi.processors.standard.http.CookieStrategy; import org.apache.nifi.processors.standard.http.CookieStrategy;
import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.reporting.InitializationException;
@ -44,6 +47,7 @@ import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException; import java.security.GeneralSecurityException;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
@ -55,6 +59,7 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
@ -101,6 +106,8 @@ public class InvokeHTTPTest {
private static final String CONTENT_LENGTH_HEADER = "Content-Length"; private static final String CONTENT_LENGTH_HEADER = "Content-Length";
private static final String CONTENT_ENCODING_HEADER = "Content-Encoding";
private static final String CONTENT_TYPE_HEADER = "Content-Type"; private static final String CONTENT_TYPE_HEADER = "Content-Type";
private static final String LOCATION_HEADER = "Location"; private static final String LOCATION_HEADER = "Location";
@ -521,7 +528,7 @@ public class InvokeHTTPTest {
final String authorization = request.getHeader(AUTHORIZATION_HEADER); final String authorization = request.getHeader(AUTHORIZATION_HEADER);
assertNotNull(authorization, "Authorization Header not found"); assertNotNull(authorization, "Authorization Header not found");
final Pattern basicAuthPattern = Pattern.compile("^Basic [^\\s]+$"); final Pattern basicAuthPattern = Pattern.compile("^Basic \\S+$");
assertTrue(basicAuthPattern.matcher(authorization).matches(), "Basic Authentication not matched"); assertTrue(basicAuthPattern.matcher(authorization).matches(), "Basic Authentication not matched");
} }
@ -723,6 +730,31 @@ public class InvokeHTTPTest {
assertRequestMethodSuccess(POST_METHOD); assertRequestMethodSuccess(POST_METHOD);
} }
@Test
public void testRunPostHttp200SuccessContentEncodingGzip() throws InterruptedException, IOException {
runner.setProperty(InvokeHTTP.PROP_METHOD, POST_METHOD);
runner.setProperty(InvokeHTTP.PROP_CONTENT_ENCODING, ContentEncodingStrategy.GZIP.getValue());
runner.setProperty(InvokeHTTP.PROP_SEND_BODY, Boolean.TRUE.toString());
enqueueResponseCodeAndRun(HTTP_OK);
assertResponseSuccessRelationships();
assertRelationshipStatusCodeEquals(InvokeHTTP.REL_RESPONSE, HTTP_OK);
final RecordedRequest request = takeRequestCompleted();
final String contentLength = request.getHeader(CONTENT_LENGTH_HEADER);
assertNull(contentLength, "Content-Length Request Header found");
final String contentEncoding = request.getHeader(CONTENT_ENCODING_HEADER);
assertEquals(ContentEncodingStrategy.GZIP.getValue().toLowerCase(), contentEncoding);
final Buffer body = request.getBody();
try (final GZIPInputStream gzipInputStream = new GZIPInputStream(body.inputStream())) {
final String decompressed = IOUtils.toString(gzipInputStream, StandardCharsets.UTF_8);
assertEquals(FLOW_FILE_CONTENT, decompressed);
}
}
@Test @Test
public void testRunPostHttp200SuccessChunkedEncoding() throws InterruptedException { public void testRunPostHttp200SuccessChunkedEncoding() throws InterruptedException {
runner.setProperty(InvokeHTTP.PROP_METHOD, POST_METHOD); runner.setProperty(InvokeHTTP.PROP_METHOD, POST_METHOD);

View File

@ -16,6 +16,7 @@
*/ */
package org.apache.nifi.processors.standard; package org.apache.nifi.processors.standard;
import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
@ -45,8 +46,12 @@ import okhttp3.OkHttpClient;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.RequestBody; import okhttp3.RequestBody;
import okhttp3.Response; import okhttp3.Response;
import okio.BufferedSink;
import okio.GzipSink;
import okio.Okio;
import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processors.standard.http.ContentEncodingStrategy;
import org.apache.nifi.processors.standard.http.HttpProtocolStrategy; import org.apache.nifi.processors.standard.http.HttpProtocolStrategy;
import org.apache.nifi.remote.io.socket.NetworkUtils; import org.apache.nifi.remote.io.socket.NetworkUtils;
import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.reporting.InitializationException;
@ -54,6 +59,7 @@ import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration; import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TemporaryKeyStoreBuilder; import org.apache.nifi.security.util.TemporaryKeyStoreBuilder;
import org.apache.nifi.security.util.TlsConfiguration; import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.security.util.TlsPlatform;
import org.apache.nifi.serialization.record.MockRecordParser; import org.apache.nifi.serialization.record.MockRecordParser;
import org.apache.nifi.serialization.record.MockRecordWriter; import org.apache.nifi.serialization.record.MockRecordWriter;
import org.apache.nifi.serialization.record.RecordFieldType; import org.apache.nifi.serialization.record.RecordFieldType;
@ -65,17 +71,17 @@ import org.apache.nifi.util.TestRunners;
import org.apache.nifi.web.util.ssl.SslContextUtils; import org.apache.nifi.web.util.ssl.SslContextUtils;
import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.util.thread.ThreadPool; import org.eclipse.jetty.util.thread.ThreadPool;
import org.junit.After; import org.junit.jupiter.api.AfterEach;
import org.junit.Assert; import org.junit.jupiter.api.BeforeEach;
import org.junit.Assume; import org.junit.jupiter.api.BeforeAll;
import org.junit.Before; import org.junit.jupiter.api.Test;
import org.junit.BeforeClass; import org.junit.jupiter.api.condition.EnabledIf;
import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import static org.apache.nifi.processors.standard.ListenHTTP.RELATIONSHIP_SUCCESS; import static org.apache.nifi.processors.standard.ListenHTTP.RELATIONSHIP_SUCCESS;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestListenHTTP { public class TestListenHTTP {
@ -114,7 +120,11 @@ public class TestListenHTTP {
private int availablePort; private int availablePort;
@BeforeClass static boolean isTls13Supported() {
return TLS_1_3.equals(TlsPlatform.getLatestProtocol());
}
@BeforeAll
public static void setUpSuite() throws GeneralSecurityException { public static void setUpSuite() throws GeneralSecurityException {
// generate new keystore and truststore // generate new keystore and truststore
final TlsConfiguration tlsConfiguration = new TemporaryKeyStoreBuilder().build(); final TlsConfiguration tlsConfiguration = new TemporaryKeyStoreBuilder().build();
@ -172,7 +182,7 @@ public class TestListenHTTP {
); );
} }
@Before @BeforeEach
public void setup() throws IOException { public void setup() throws IOException {
proc = new ListenHTTP(); proc = new ListenHTTP();
runner = TestRunners.newTestRunner(proc); runner = TestRunners.newTestRunner(proc);
@ -181,7 +191,7 @@ public class TestListenHTTP {
runner.setVariable(BASEPATH_VARIABLE, HTTP_BASE_PATH); runner.setVariable(BASEPATH_VARIABLE, HTTP_BASE_PATH);
} }
@After @AfterEach
public void shutdownServer() { public void shutdownServer() {
proc.shutdownHttpServer(); proc.shutdownHttpServer();
} }
@ -360,13 +370,14 @@ public class TestListenHTTP {
startSecureServer(); startSecureServer();
final SSLSocketFactory sslSocketFactory = trustStoreSslContext.getSocketFactory(); final SSLSocketFactory sslSocketFactory = trustStoreSslContext.getSocketFactory();
final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort); try (final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort)) {
final String currentProtocol = serverNoTruststoreConfiguration.getProtocol(); final String currentProtocol = serverNoTruststoreConfiguration.getProtocol();
sslSocket.setEnabledProtocols(new String[]{currentProtocol}); sslSocket.setEnabledProtocols(new String[]{currentProtocol});
sslSocket.startHandshake(); sslSocket.startHandshake();
final SSLSession sslSession = sslSocket.getSession(); final SSLSession sslSession = sslSocket.getSession();
assertEquals("SSL Session Protocol not matched", currentProtocol, sslSession.getProtocol()); assertEquals(currentProtocol, sslSession.getProtocol());
}
} }
@Test @Test
@ -384,12 +395,9 @@ public class TestListenHTTP {
assertEquals(HttpServletResponse.SC_NO_CONTENT, responseCode); assertEquals(HttpServletResponse.SC_NO_CONTENT, responseCode);
} }
@EnabledIf(value = "isTls13Supported", disabledReason = "TLSv1.3 is not supported")
@Test @Test
public void testSecureServerRejectsUnsupportedTlsProtocolVersion() throws Exception { public void testSecureServerRejectsUnsupportedTlsProtocolVersion() throws Exception {
final String currentProtocol = TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion();
final String protocolMessage = String.format("TLS Protocol required [%s] found [%s]", TLS_1_3, currentProtocol);
Assume.assumeTrue(protocolMessage, TLS_1_3.equals(currentProtocol));
configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, serverTls_1_3_Configuration); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, serverTls_1_3_Configuration);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
@ -399,68 +407,58 @@ public class TestListenHTTP {
startWebServer(); startWebServer();
final SSLSocketFactory sslSocketFactory = trustStoreSslContext.getSocketFactory(); final SSLSocketFactory sslSocketFactory = trustStoreSslContext.getSocketFactory();
final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort); try (final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort)) {
sslSocket.setEnabledProtocols(new String[]{TLS_1_2}); sslSocket.setEnabledProtocols(new String[]{TLS_1_2});
assertThrows(SSLHandshakeException.class, sslSocket::startHandshake); assertThrows(SSLHandshakeException.class, sslSocket::startHandshake);
}
} }
@Test @Test
public void testMaxThreadPoolSizeTooLow() { public void testMaxThreadPoolSizeTooLow() {
// GIVEN, WHEN
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "7"); runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "7");
// THEN
runner.assertNotValid(); runner.assertNotValid();
} }
@Test @Test
public void testMaxThreadPoolSizeTooHigh() { public void testMaxThreadPoolSizeTooHigh() {
// GIVEN, WHEN
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "1001"); runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "1001");
// THEN
runner.assertNotValid(); runner.assertNotValid();
} }
@Test @Test
public void testMaxThreadPoolSizeOkLowerBound() { public void testMaxThreadPoolSizeOkLowerBound() {
// GIVEN, WHEN
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "8"); runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "8");
// THEN
runner.assertValid(); runner.assertValid();
} }
@Test @Test
public void testMaxThreadPoolSizeOkUpperBound() { public void testMaxThreadPoolSizeOkUpperBound() {
// GIVEN, WHEN
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "1000"); runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, "1000");
// THEN
runner.assertValid(); runner.assertValid();
} }
@Test @Test
public void testMaxThreadPoolSizeSpecifiedInThePropertyIsSetInTheServerInstance() { public void testMaxThreadPoolSizeSpecifiedInThePropertyIsSetInTheServerInstance() {
// GIVEN
int maxThreadPoolSize = 201; int maxThreadPoolSize = 201;
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, Integer.toString(maxThreadPoolSize)); runner.setProperty(ListenHTTP.MAX_THREAD_POOL_SIZE, Integer.toString(maxThreadPoolSize));
// WHEN
startWebServer(); startWebServer();
// THEN
Server server = proc.getServer(); Server server = proc.getServer();
ThreadPool threadPool = server.getThreadPool(); ThreadPool threadPool = server.getThreadPool();
ThreadPool.SizedThreadPool sizedThreadPool = (ThreadPool.SizedThreadPool) threadPool; ThreadPool.SizedThreadPool sizedThreadPool = (ThreadPool.SizedThreadPool) threadPool;
@ -518,6 +516,40 @@ public class TestListenHTTP {
runner.assertTransferCount(RELATIONSHIP_SUCCESS, 0); runner.assertTransferCount(RELATIONSHIP_SUCCESS, 0);
} }
@Test
public void testPostContentEncodingGzipAccepted() throws IOException {
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.RETURN_CODE, Integer.toString(HttpServletResponse.SC_NO_CONTENT));
startWebServer();
final OkHttpClient okHttpClient = getOkHttpClient(false, false);
final Request.Builder requestBuilder = new Request.Builder();
final String url = buildUrl(false);
requestBuilder.url(url);
final String message = String.class.getSimpleName();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final BufferedSink gzipSink = Okio.buffer(new GzipSink(Okio.sink(outputStream)));
gzipSink.write(message.getBytes(StandardCharsets.UTF_8));
gzipSink.close();
final byte[] compressed = outputStream.toByteArray();
final RequestBody requestBody = RequestBody.create(compressed, APPLICATION_OCTET_STREAM);
final Request request = requestBuilder.post(requestBody)
.addHeader("Content-Encoding", ContentEncodingStrategy.GZIP.getValue().toLowerCase())
.build();
try (final Response response = okHttpClient.newCall(request).execute()) {
assertTrue(response.isSuccessful());
runner.assertTransferCount(RELATIONSHIP_SUCCESS, 1);
final MockFlowFile flowFile = runner.getFlowFilesForRelationship(RELATIONSHIP_SUCCESS).iterator().next();
flowFile.assertContentEquals(message);
}
}
private MockRecordParser setupRecordReaderTest() throws InitializationException { private MockRecordParser setupRecordReaderTest() throws InitializationException {
final MockRecordParser parser = new MockRecordParser(); final MockRecordParser parser = new MockRecordParser();
final MockRecordWriter writer = new MockRecordWriter(); final MockRecordWriter writer = new MockRecordWriter();
@ -636,7 +668,7 @@ public class TestListenHTTP {
for (final String message : messages) { for (final String message : messages) {
final int statusCode = postMessage(message, secure, clientAuthRequired); final int statusCode = postMessage(message, secure, clientAuthRequired);
assertEquals("HTTP Status Code not matched", expectedStatusCode, statusCode); assertEquals(expectedStatusCode, statusCode, "HTTP Status Code not matched");
} }
} }
@ -690,7 +722,7 @@ public class TestListenHTTP {
try (Response response = client.newCall(request).execute()) { try (Response response = client.newCall(request).execute()) {
Files.deleteIfExists(Paths.get(String.valueOf(file1))); Files.deleteIfExists(Paths.get(String.valueOf(file1)));
Files.deleteIfExists(Paths.get(String.valueOf(file2))); Files.deleteIfExists(Paths.get(String.valueOf(file2)));
Assert.assertTrue(String.format("Unexpected code: %s, body: %s", response.code(), response.body()), response.isSuccessful()); assertTrue(response.isSuccessful(), String.format("Unexpected code: %s, body: %s", response.code(), response.body()));
} }
runner.assertAllFlowFilesTransferred(ListenHTTP.RELATIONSHIP_SUCCESS, 5); runner.assertAllFlowFilesTransferred(ListenHTTP.RELATIONSHIP_SUCCESS, 5);