Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-8171 This closes #4779. Upgraded Bouncy Castle libraries to 1.68 and centralized dependency version 

NIFI-8171 Increased response and idle timeouts for HTTP unit tests
NIFI-8171 Increased TestServer idle timeout to 45 seconds for HTTP unit tests
NIFI-8171 Adjusted timeout and sleep on TestPutTCPCommon.testPruneSenders
NIFI-8171 Increased TestServer idle timeout to 60 seconds and removed 500ms Thread.sleep() in TestInvokeHttpSSL
NIFI-8171 Optimized OkHttpClientUtils to avoid reading trust store twice during initialization
NIFI-8171 Added static variable for server startup sleep
NIFI-8171 Increased TestInvokeHTTP Connect Timeout and TestListenHTTP Response Timeout to 30 seconds
NIFI-8171 Refactored unit tests for InvokeHTTP and ListenHTTP to optimize SSLContext creation
NIFI-8171 Updated TestListenHTTP for static creation of SSLContext
NIFI-8171 Added started check for ListenHTTP Server in TestListenHTTP
NIFI-8171 Refactored TestPutTCP classes to optimize SSLContext creation
NIFI-8171 Increased TestListenHTTP timeout for server start to 120 seconds and added exception when not connected
NIFI-8171 Increased Connect and Read Timeouts for InvokeHTTP SSL unit tests

Signed-off-by: Joe Witt <joewitt@apache.org>
This commit is contained in:
exceptionfactory 2021-01-25 07:42:06 -06:00 committed by Joe Witt
parent 2cdb0fb6a3
commit abb6ed3128
No known key found for this signature in database
GPG Key ID: 9093BF854F811A1A
21 changed files with 455 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nifi-commons/nifi-data-provenance-utils/pom.xml
View File

@ -45,7 +45,6 @@
<dependency> <dependency>
<groupId>org.bouncycastle</groupId> <groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId> <artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.nifi</groupId> <groupId>org.apache.nifi</groupId>

2
nifi-commons/nifi-security-utils/pom.xml
View File

@ -60,12 +60,10 @@
<dependency> <dependency>
<groupId>org.bouncycastle</groupId> <groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId> <artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.bouncycastle</groupId> <groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId> <artifactId>bcpkix-jdk15on</artifactId>
<version>1.66</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.nifi</groupId> <groupId>org.apache.nifi</groupId>

19
nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/security/util/OkHttpClientUtils.java
View File

@ -17,7 +17,9 @@
package org.apache.nifi.security.util; package org.apache.nifi.security.util;
import java.security.UnrecoverableKeyException; import java.security.UnrecoverableKeyException;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager; import javax.net.ssl.X509TrustManager;
import okhttp3.OkHttpClient; import okhttp3.OkHttpClient;
import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.exception.ProcessException;
@ -41,12 +43,19 @@ public class OkHttpClientUtils {
*/ */
public static boolean applyTlsToOkHttpClientBuilder(TlsConfiguration tlsConfiguration, OkHttpClient.Builder okHttpClient) { public static boolean applyTlsToOkHttpClientBuilder(TlsConfiguration tlsConfiguration, OkHttpClient.Builder okHttpClient) {
try { try {
final SSLSocketFactory sslSocketFactory = SslContextFactory.createSSLSocketFactory(tlsConfiguration); final X509TrustManager trustManager = SslContextFactory.getX509TrustManager(tlsConfiguration);
final X509TrustManager x509TrustManager = SslContextFactory.getX509TrustManager(tlsConfiguration); if (trustManager == null) {
if (sslSocketFactory != null && x509TrustManager != null) { return false;
okHttpClient.sslSocketFactory(sslSocketFactory, x509TrustManager);
return true;
} }
final SSLContext sslContext = SslContextFactory.createSslContext(tlsConfiguration, new TrustManager[]{trustManager});
if (sslContext == null) {
return false;
}
final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();
okHttpClient.sslSocketFactory(sslSocketFactory, trustManager);
return true;
} catch (TlsException e) { } catch (TlsException e) {
if (e.getCause() instanceof UnrecoverableKeyException) { if (e.getCause() instanceof UnrecoverableKeyException) {
logger.error("Key password may be incorrect or not set. Check your keystore passwords." + e.getMessage()); logger.error("Key password may be incorrect or not set. Check your keystore passwords." + e.getMessage());

1
nifi-nar-bundles/nifi-druid-bundle/nifi-druid-controller-service-api/pom.xml
View File

@ -135,7 +135,6 @@
<dependency> <dependency>
<groupId>org.bouncycastle</groupId> <groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId> <artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
</dependencies> </dependencies>

10
nifi-nar-bundles/nifi-framework-bundle/pom.xml
View File

@ -320,16 +320,6 @@
<artifactId>h2</artifactId> <artifactId>h2</artifactId>
<version>1.4.199</version> <version>1.4.199</version>
</dependency> </dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.66</version>
</dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>

5
nifi-nar-bundles/nifi-provenance-repository-bundle/pom.xml
View File

@ -61,11 +61,6 @@
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.9</version> <version>3.9</version>
</dependency> </dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
</project> </project>

19
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
View File

@ -31,12 +31,7 @@ import java.net.Proxy.Type;
import java.net.URL; import java.net.URL;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.Principal; import java.security.Principal;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@ -55,7 +50,11 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.X509TrustManager;
import okhttp3.Cache; import okhttp3.Cache;
import okhttp3.ConnectionPool; import okhttp3.ConnectionPool;
import okhttp3.Credentials; import okhttp3.Credentials;
@ -100,8 +99,9 @@ import org.apache.nifi.processors.standard.util.ProxyAuthenticator;
import org.apache.nifi.processors.standard.util.SoftLimitBoundedByteArrayOutputStream; import org.apache.nifi.processors.standard.util.SoftLimitBoundedByteArrayOutputStream;
import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxyConfiguration;
import org.apache.nifi.proxy.ProxySpec; import org.apache.nifi.proxy.ProxySpec;
import org.apache.nifi.security.util.OkHttpClientUtils; import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.TlsConfiguration; import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.ssl.SSLContextService; import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.stream.io.StreamUtils; import org.apache.nifi.stream.io.StreamUtils;
import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormat;
@ -704,7 +704,7 @@ public class InvokeHTTP extends AbstractProcessor {
} }
@OnScheduled @OnScheduled
public void setUpClient(final ProcessContext context) throws IOException, UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { public void setUpClient(final ProcessContext context) throws TlsException {
okHttpClientAtomicReference.set(null); okHttpClientAtomicReference.set(null);
OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient().newBuilder(); OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient().newBuilder();
@ -761,8 +761,11 @@ public class InvokeHTTP extends AbstractProcessor {
// Apply the TLS configuration if present // Apply the TLS configuration if present
final SSLContextService sslService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); final SSLContextService sslService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
if (sslService != null) { if (sslService != null) {
final SSLContext sslContext = sslService.createContext();
final SSLSocketFactory socketFactory = sslContext.getSocketFactory();
final TlsConfiguration tlsConfiguration = sslService.createTlsConfiguration(); final TlsConfiguration tlsConfiguration = sslService.createTlsConfiguration();
OkHttpClientUtils.applyTlsToOkHttpClientBuilder(tlsConfiguration, okHttpClientBuilder); final X509TrustManager trustManager = SslContextFactory.getX509TrustManager(tlsConfiguration);
okHttpClientBuilder.sslSocketFactory(socketFactory, trustManager);
} }
setAuthenticator(okHttpClientBuilder, context); setAuthenticator(okHttpClientBuilder, context);

37
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java
View File

@ -31,18 +31,13 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.SystemUtils;
import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon; import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon;
import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.ssl.StandardSSLContextService;
import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunners; import org.apache.nifi.util.TestRunners;
import org.apache.nifi.web.util.TestServer;
import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.server.handler.AbstractHandler;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
@ -54,40 +49,12 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception { public static void beforeClass() throws Exception {
Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS); configureServer(null, null);
// useful for verbose logging output
// don't commit this with this property enabled, or any 'mvn test' will be really verbose
// System.setProperty("org.slf4j.simpleLogger.log.nifi.processors.standard", "debug");
// create a Jetty server on a random port
server = createServer();
server.startServer();
// this is the base url with the random port
url = server.getUrl();
}
@AfterClass
public static void afterClass() throws Exception {
if(server != null) {
server.shutdownServer();
}
} }
@Before @Before
public void before() { public void before() throws Exception {
runner = TestRunners.newTestRunner(InvokeHTTP.class); runner = TestRunners.newTestRunner(InvokeHTTP.class);
server.clearHandlers();
}
@After
public void after() {
runner.shutdown();
}
private static TestServer createServer() {
return new TestServer();
} }
@Test @Test

164
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java
View File

@ -17,20 +17,20 @@
package org.apache.nifi.processors.standard; package org.apache.nifi.processors.standard;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.SystemUtils;
import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon; import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon;
import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.security.util.ClientAuth;
import org.apache.nifi.security.util.KeystoreType;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.util.TestRunners; import org.apache.nifi.util.TestRunners;
import org.apache.nifi.web.util.TestServer;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.mockito.Mockito;
import javax.net.ssl.SSLContext;
/** /**
* Executes the same tests as TestInvokeHttp but with one-way SSL enabled. The Jetty server created for these tests * Executes the same tests as TestInvokeHttp but with one-way SSL enabled. The Jetty server created for these tests
@ -38,115 +38,59 @@ import org.junit.BeforeClass;
*/ */
public class TestInvokeHttpSSL extends TestInvokeHttpCommon { public class TestInvokeHttpSSL extends TestInvokeHttpCommon {
protected static Map<String, String> sslProperties; protected static final String TRUSTSTORE_PATH = "src/test/resources/truststore.no-password.jks";
protected static Map<String, String> serverSslProperties; protected static final String TRUSTSTORE_PASSWORD = "";
protected static final KeystoreType TRUSTSTORE_TYPE = KeystoreType.JKS;
private static final String KEYSTORE_PATH = "src/test/resources/keystore.jks";
private static final String KEYSTORE_PASSWORD = "passwordpassword";
private static final KeystoreType KEYSTORE_TYPE = KeystoreType.JKS;
private static final String HTTP_CONNECT_TIMEOUT = "30 s";
private static final String HTTP_READ_TIMEOUT = "30 s";
protected static final TlsConfiguration SERVER_CONFIGURATION = new StandardTlsConfiguration(
KEYSTORE_PATH,
KEYSTORE_PASSWORD,
KEYSTORE_TYPE,
TRUSTSTORE_PATH,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE
);
protected static SSLContext clientSslContext;
private static final TlsConfiguration CLIENT_CONFIGURATION = new StandardTlsConfiguration(
null,
null,
null,
TRUSTSTORE_PATH,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE
);
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception { public static void beforeClass() throws Exception {
Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS); final SSLContext serverContext = SslContextFactory.createSslContext(SERVER_CONFIGURATION);
// useful for verbose logging output configureServer(serverContext, ClientAuth.NONE);
// don't commit this with this property enabled, or any 'mvn test' will be really verbose clientSslContext = SslContextFactory.createSslContext(CLIENT_CONFIGURATION);
// System.setProperty("org.slf4j.simpleLogger.log.nifi.processors.standard", "debug");
// create the SSL properties, which basically store keystore / truststore information
// this is used by the StandardSSLContextService and the Jetty Server
serverSslProperties = createServerSslProperties(false);
sslProperties = createClientSslProperties(false);
// create a Jetty server on a random port
server = createServer();
server.startServer();
// Allow time for the server to start
Thread.sleep(500);
// this is the base url with the random port
url = server.getSecureUrl();
}
@AfterClass
public static void afterClass() throws Exception {
if(server != null) {
server.shutdownServer();
}
} }
@Before @Before
public void before() throws Exception { public void before() throws Exception {
final SSLContextService sslContextService = Mockito.mock(SSLContextService.class);
final String serviceIdentifier = SSLContextService.class.getName();
Mockito.when(sslContextService.getIdentifier()).thenReturn(serviceIdentifier);
Mockito.when(sslContextService.createContext()).thenReturn(clientSslContext);
Mockito.when(sslContextService.createTlsConfiguration()).thenReturn(CLIENT_CONFIGURATION);
runner = TestRunners.newTestRunner(InvokeHTTP.class); runner = TestRunners.newTestRunner(InvokeHTTP.class);
final StandardSSLContextService sslService = new StandardSSLContextService(); runner.addControllerService(serviceIdentifier, sslContextService);
runner.addControllerService("ssl-context", sslService, sslProperties); runner.enableControllerService(sslContextService);
runner.enableControllerService(sslService);
runner.setProperty(InvokeHTTP.PROP_SSL_CONTEXT_SERVICE, "ssl-context");
// Allow time for the controller service to fully initialize runner.setProperty(InvokeHTTP.PROP_SSL_CONTEXT_SERVICE, serviceIdentifier);
Thread.sleep(500); runner.setProperty(InvokeHTTP.PROP_CONNECT_TIMEOUT, HTTP_CONNECT_TIMEOUT);
runner.setProperty(InvokeHTTP.PROP_READ_TIMEOUT, HTTP_READ_TIMEOUT);
// Provide more time to setup and run
runner.setProperty(InvokeHTTP.PROP_READ_TIMEOUT, "30 secs");
runner.setProperty(InvokeHTTP.PROP_CONNECT_TIMEOUT, "30 secs");
server.clearHandlers();
}
@After
public void after() {
runner.shutdown();
}
static TestServer createServer() throws IOException {
return new TestServer(serverSslProperties);
}
static Map<String, String> createServerSslProperties(boolean clientAuth) {
final Map<String, String> map = new HashMap<>();
// if requesting client auth then we must also provide a truststore
if (clientAuth) {
map.put(TestServer.NEED_CLIENT_AUTH, Boolean.toString(true));
map.putAll(getTruststoreProperties());
} else {
map.put(TestServer.NEED_CLIENT_AUTH, Boolean.toString(false));
}
// keystore is always required for the server SSL properties
map.putAll(getServerKeystoreProperties());
return map;
}
static Map<String, String> createClientSslProperties(boolean clientAuth) {
final Map<String, String> map = new HashMap<>();
// if requesting client auth then we must provide a keystore
if (clientAuth) {
map.putAll(getClientKeystoreProperties());
}
// truststore is always required for the client SSL properties
map.putAll(getTruststoreProperties());
return map;
}
private static Map<String, String> getServerKeystoreProperties() {
final Map<String, String> map = new HashMap<>();
map.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/keystore.jks");
map.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "passwordpassword");
map.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
return map;
}
private static Map<String, String> getClientKeystoreProperties() {
final Map<String, String> map = new HashMap<>();
map.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/client-keystore.p12");
map.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "passwordpassword");
map.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "PKCS12");
return map;
}
private static Map<String, String> getTruststoreProperties() {
final Map<String, String> map = new HashMap<>();
map.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/truststore.no-password.jks");
// Commented this line to test passwordless truststores for NIFI-6770
// map.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "passwordpassword");
map.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
return map;
} }
} }

43
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpTwoWaySSL.java
View File

@ -17,10 +17,15 @@
package org.apache.nifi.processors.standard; package org.apache.nifi.processors.standard;
import org.apache.commons.lang3.SystemUtils; import org.apache.nifi.security.util.ClientAuth;
import org.junit.Assume; import org.apache.nifi.security.util.KeystoreType;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TlsConfiguration;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import javax.net.ssl.SSLContext;
/** /**
* This is probably overkill but in keeping with the same pattern as the TestInvokeHttp and TestInvokeHttpSSL class, * This is probably overkill but in keeping with the same pattern as the TestInvokeHttp and TestInvokeHttpSSL class,
* we will execute the same tests using two-way SSL. The Jetty server created for these tests will require client * we will execute the same tests using two-way SSL. The Jetty server created for these tests will require client
@ -28,27 +33,23 @@ import org.junit.BeforeClass;
*/ */
public class TestInvokeHttpTwoWaySSL extends TestInvokeHttpSSL { public class TestInvokeHttpTwoWaySSL extends TestInvokeHttpSSL {
private static final String CLIENT_KEYSTORE_PATH = "src/test/resources/client-keystore.p12";
private static final String CLIENT_KEYSTORE_PASSWORD = "passwordpassword";
private static final KeystoreType CLIENT_KEYSTORE_TYPE = KeystoreType.PKCS12;
private static final TlsConfiguration CLIENT_CONFIGURATION = new StandardTlsConfiguration(
CLIENT_KEYSTORE_PATH,
CLIENT_KEYSTORE_PASSWORD,
CLIENT_KEYSTORE_TYPE,
TRUSTSTORE_PATH,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE
);
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception { public static void beforeClass() throws Exception {
Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS); final SSLContext serverContext = SslContextFactory.createSslContext(SERVER_CONFIGURATION);
// useful for verbose logging output configureServer(serverContext, ClientAuth.REQUIRED);
// don't commit this with this property enabled, or any 'mvn test' will be really verbose clientSslContext = SslContextFactory.createSslContext(CLIENT_CONFIGURATION);
// System.setProperty("org.slf4j.simpleLogger.log.nifi.processors.standard", "debug");
// create the SSL properties, which basically store keystore / trustore information
// this is used by the StandardSSLContextService and the Jetty Server
serverSslProperties = createServerSslProperties(true);
sslProperties = createClientSslProperties(true);
// create a Jetty server on a random port
server = createServer();
server.startServer();
// Allow time for the server to start
Thread.sleep(500);
// this is the base url with the random port
url = server.getSecureUrl();
} }
} }

226
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListenHTTP.java
View File

@ -29,6 +29,8 @@ import java.io.DataOutputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -41,6 +43,7 @@ import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import okhttp3.MediaType; import okhttp3.MediaType;
import okhttp3.MultipartBody; import okhttp3.MultipartBody;
@ -48,7 +51,7 @@ import okhttp3.OkHttpClient;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.RequestBody; import okhttp3.RequestBody;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.SystemUtils;
import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.remote.io.socket.NetworkUtils; import org.apache.nifi.remote.io.socket.NetworkUtils;
@ -58,8 +61,8 @@ import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration; import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TlsConfiguration; import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.security.util.TlsException; import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.ssl.RestrictedSSLContextService;
import org.apache.nifi.ssl.SSLContextService; import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.ssl.StandardRestrictedSSLContextService;
import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.ssl.StandardSSLContextService;
import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunner;
@ -70,6 +73,7 @@ import org.junit.Assume;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito;
public class TestListenHTTP { public class TestListenHTTP {
@ -86,18 +90,57 @@ public class TestListenHTTP {
private static final String KEYSTORE = "src/test/resources/keystore.jks"; private static final String KEYSTORE = "src/test/resources/keystore.jks";
private static final String KEYSTORE_PASSWORD = "passwordpassword"; private static final String KEYSTORE_PASSWORD = "passwordpassword";
private static final KeystoreType KEYSTORE_TYPE = KeystoreType.JKS;
private static final String TRUSTSTORE = "src/test/resources/truststore.jks"; private static final String TRUSTSTORE = "src/test/resources/truststore.jks";
private static final String TRUSTSTORE_PASSWORD = "passwordpassword"; private static final String TRUSTSTORE_PASSWORD = "passwordpassword";
private static final String TRUSTSTORE_TYPE = KeystoreType.JKS.getType(); private static final KeystoreType TRUSTSTORE_TYPE = KeystoreType.JKS;
private static final String CLIENT_KEYSTORE = "src/test/resources/client-keystore.p12"; private static final String CLIENT_KEYSTORE = "src/test/resources/client-keystore.p12";
private static final String CLIENT_KEYSTORE_TYPE = KeystoreType.PKCS12.getType(); private static final KeystoreType CLIENT_KEYSTORE_TYPE = KeystoreType.PKCS12;
private static final String TLS_1_3 = "TLSv1.3"; private static final String TLS_1_3 = "TLSv1.3";
private static final String TLS_1_2 = "TLSv1.2"; private static final String TLS_1_2 = "TLSv1.2";
private static final String LOCALHOST = "localhost"; private static final String LOCALHOST = "localhost";
private static TlsConfiguration clientTlsConfiguration; private static final long SEND_REQUEST_SLEEP = 150;
private static TlsConfiguration trustOnlyTlsConfiguration; private static final long RESPONSE_TIMEOUT = 1200000;
private static final int SOCKET_CONNECT_TIMEOUT = 100;
private static final long SERVER_START_TIMEOUT = 1200000;
private static final TlsConfiguration SERVER_CONFIGURATION = new StandardTlsConfiguration(
KEYSTORE,
KEYSTORE_PASSWORD,
KEYSTORE_PASSWORD,
KEYSTORE_TYPE,
TRUSTSTORE,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE,
TLS_1_2
);
private static final TlsConfiguration SERVER_TLS_1_3_CONFIGURATION = new StandardTlsConfiguration(
KEYSTORE,
KEYSTORE_PASSWORD,
KEYSTORE_PASSWORD,
KEYSTORE_TYPE,
TRUSTSTORE,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE,
TLS_1_3
);
private static final TlsConfiguration SERVER_NO_TRUSTSTORE_CONFIGURATION = new StandardTlsConfiguration(
KEYSTORE,
KEYSTORE_PASSWORD,
KEYSTORE_PASSWORD,
KEYSTORE_TYPE,
null,
null,
null,
TLS_1_2
);
private static SSLContext serverKeyStoreSslContext;
private static SSLContext serverKeyStoreNoTrustStoreSslContext;
private static SSLContext keyStoreSslContext;
private static SSLContext trustStoreSslContext;
private ListenHTTP proc; private ListenHTTP proc;
private TestRunner runner; private TestRunner runner;
@ -105,8 +148,27 @@ public class TestListenHTTP {
private int availablePort; private int availablePort;
@BeforeClass @BeforeClass
public static void setUpSuite() { public static void setUpSuite() throws TlsException {
Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS); serverKeyStoreSslContext = SslContextFactory.createSslContext(SERVER_CONFIGURATION);
final TrustManager[] defaultTrustManagers = SslContextFactory.getTrustManagers(SERVER_NO_TRUSTSTORE_CONFIGURATION);
serverKeyStoreNoTrustStoreSslContext = SslContextFactory.createSslContext(SERVER_NO_TRUSTSTORE_CONFIGURATION, defaultTrustManagers);
keyStoreSslContext = SslContextFactory.createSslContext(new StandardTlsConfiguration(
CLIENT_KEYSTORE,
KEYSTORE_PASSWORD,
CLIENT_KEYSTORE_TYPE,
TRUSTSTORE,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE)
);
trustStoreSslContext = SslContextFactory.createSslContext(new StandardTlsConfiguration(
null,
null,
null,
TRUSTSTORE,
TRUSTSTORE_PASSWORD,
TRUSTSTORE_TYPE)
);
} }
@Before @Before
@ -116,11 +178,6 @@ public class TestListenHTTP {
availablePort = NetworkUtils.availablePort(); availablePort = NetworkUtils.availablePort();
runner.setVariable(PORT_VARIABLE, Integer.toString(availablePort)); runner.setVariable(PORT_VARIABLE, Integer.toString(availablePort));
runner.setVariable(BASEPATH_VARIABLE, HTTP_BASE_PATH); runner.setVariable(BASEPATH_VARIABLE, HTTP_BASE_PATH);
clientTlsConfiguration = new StandardTlsConfiguration(CLIENT_KEYSTORE, KEYSTORE_PASSWORD, null, CLIENT_KEYSTORE_TYPE,
TRUSTSTORE, TRUSTSTORE_PASSWORD, TRUSTSTORE_TYPE, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
trustOnlyTlsConfiguration = new StandardTlsConfiguration(null, null, null, null,
TRUSTSTORE, TRUSTSTORE_PASSWORD, TRUSTSTORE_TYPE, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
} }
@After @After
@ -167,9 +224,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecurePOSTRequestsReceivedWithoutEL() throws Exception { public void testSecurePOSTRequestsReceivedWithoutEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_NO_TRUSTSTORE_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -180,9 +235,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecurePOSTRequestsReturnCodeReceivedWithoutEL() throws Exception { public void testSecurePOSTRequestsReturnCodeReceivedWithoutEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_NO_TRUSTSTORE_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -194,9 +247,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecurePOSTRequestsReceivedWithEL() throws Exception { public void testSecurePOSTRequestsReceivedWithEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_NO_TRUSTSTORE_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, HTTP_SERVER_PORT_EL); runner.setProperty(ListenHTTP.PORT, HTTP_SERVER_PORT_EL);
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_SERVER_BASEPATH_EL); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_SERVER_BASEPATH_EL);
@ -207,9 +258,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecurePOSTRequestsReturnCodeReceivedWithEL() throws Exception { public void testSecurePOSTRequestsReturnCodeReceivedWithEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_NO_TRUSTSTORE_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -221,9 +270,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecureTwoWaySslPOSTRequestsReceivedWithoutEL() throws Exception { public void testSecureTwoWaySslPOSTRequestsReceivedWithoutEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(true); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.REQUIRED, SERVER_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -234,9 +281,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecureTwoWaySslPOSTRequestsReturnCodeReceivedWithoutEL() throws Exception { public void testSecureTwoWaySslPOSTRequestsReturnCodeReceivedWithoutEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(true); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.REQUIRED, SERVER_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -248,9 +293,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecureTwoWaySslPOSTRequestsReceivedWithEL() throws Exception { public void testSecureTwoWaySslPOSTRequestsReceivedWithEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(true); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.REQUIRED, SERVER_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, HTTP_SERVER_PORT_EL); runner.setProperty(ListenHTTP.PORT, HTTP_SERVER_PORT_EL);
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_SERVER_BASEPATH_EL); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_SERVER_BASEPATH_EL);
@ -261,9 +304,7 @@ public class TestListenHTTP {
@Test @Test
public void testSecureTwoWaySslPOSTRequestsReturnCodeReceivedWithEL() throws Exception { public void testSecureTwoWaySslPOSTRequestsReturnCodeReceivedWithEL() throws Exception {
SSLContextService sslContextService = configureProcessorSslContextService(true); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.REQUIRED, SERVER_CONFIGURATION);
runner.setProperty(sslContextService, StandardRestrictedSSLContextService.RESTRICTED_SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -276,7 +317,6 @@ public class TestListenHTTP {
@Test @Test
public void testSecureInvalidSSLConfiguration() throws Exception { public void testSecureInvalidSSLConfiguration() throws Exception {
SSLContextService sslContextService = configureInvalidProcessorSslContextService(); SSLContextService sslContextService = configureInvalidProcessorSslContextService();
runner.setProperty(sslContextService, StandardSSLContextService.SSL_ALGORITHM, TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
runner.enableControllerService(sslContextService); runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, HTTP_SERVER_PORT_EL); runner.setProperty(ListenHTTP.PORT, HTTP_SERVER_PORT_EL);
@ -286,11 +326,12 @@ public class TestListenHTTP {
@Test @Test
public void testSecureServerSupportsCurrentTlsProtocolVersion() throws Exception { public void testSecureServerSupportsCurrentTlsProtocolVersion() throws Exception {
startSecureServer(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_NO_TRUSTSTORE_CONFIGURATION);
startSecureServer();
final SSLSocketFactory sslSocketFactory = SslContextFactory.createSSLSocketFactory(trustOnlyTlsConfiguration); final SSLSocketFactory sslSocketFactory = trustStoreSslContext.getSocketFactory();
final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort); final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort);
final String currentProtocol = TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion(); final String currentProtocol = SERVER_NO_TRUSTSTORE_CONFIGURATION.getProtocol();
sslSocket.setEnabledProtocols(new String[]{currentProtocol}); sslSocket.setEnabledProtocols(new String[]{currentProtocol});
sslSocket.startHandshake(); sslSocket.startHandshake();
@ -300,19 +341,21 @@ public class TestListenHTTP {
@Test @Test
public void testSecureServerTrustStoreConfiguredClientAuthenticationRequired() throws Exception { public void testSecureServerTrustStoreConfiguredClientAuthenticationRequired() throws Exception {
startSecureServer(true); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.REQUIRED, SERVER_CONFIGURATION);
final HttpsURLConnection connection = getSecureConnection(trustOnlyTlsConfiguration); startSecureServer();
final HttpsURLConnection connection = getSecureConnection(trustStoreSslContext);
assertThrows(SSLException.class, connection::getResponseCode); assertThrows(SSLException.class, connection::getResponseCode);
final HttpsURLConnection clientCertificateConnection = getSecureConnection(clientTlsConfiguration); final HttpsURLConnection clientCertificateConnection = getSecureConnection(keyStoreSslContext);
final int responseCode = clientCertificateConnection.getResponseCode(); final int responseCode = clientCertificateConnection.getResponseCode();
assertEquals(HttpServletResponse.SC_METHOD_NOT_ALLOWED, responseCode); assertEquals(HttpServletResponse.SC_METHOD_NOT_ALLOWED, responseCode);
} }
@Test @Test
public void testSecureServerTrustStoreNotConfiguredClientAuthenticationNotRequired() throws Exception { public void testSecureServerTrustStoreNotConfiguredClientAuthenticationNotRequired() throws Exception {
startSecureServer(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_NO_TRUSTSTORE_CONFIGURATION);
final HttpsURLConnection connection = getSecureConnection(trustOnlyTlsConfiguration); startSecureServer();
final HttpsURLConnection connection = getSecureConnection(trustStoreSslContext);
final int responseCode = connection.getResponseCode(); final int responseCode = connection.getResponseCode();
assertEquals(HttpServletResponse.SC_METHOD_NOT_ALLOWED, responseCode); assertEquals(HttpServletResponse.SC_METHOD_NOT_ALLOWED, responseCode);
} }
@ -323,9 +366,7 @@ public class TestListenHTTP {
final String protocolMessage = String.format("TLS Protocol required [%s] found [%s]", TLS_1_3, currentProtocol); final String protocolMessage = String.format("TLS Protocol required [%s] found [%s]", TLS_1_3, currentProtocol);
Assume.assumeTrue(protocolMessage, TLS_1_3.equals(currentProtocol)); Assume.assumeTrue(protocolMessage, TLS_1_3.equals(currentProtocol));
final SSLContextService sslContextService = configureProcessorSslContextService(false); configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, SERVER_TLS_1_3_CONFIGURATION);
runner.setProperty(sslContextService, StandardSSLContextService.SSL_ALGORITHM, TLS_1_3);
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
@ -333,18 +374,14 @@ public class TestListenHTTP {
runner.assertValid(); runner.assertValid();
startWebServer(); startWebServer();
final SSLSocketFactory sslSocketFactory = SslContextFactory.createSSLSocketFactory(trustOnlyTlsConfiguration); final SSLSocketFactory sslSocketFactory = trustStoreSslContext.getSocketFactory();
final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort); final SSLSocket sslSocket = (SSLSocket) sslSocketFactory.createSocket(LOCALHOST, availablePort);
sslSocket.setEnabledProtocols(new String[]{TLS_1_2}); sslSocket.setEnabledProtocols(new String[]{TLS_1_2});
assertThrows(SSLHandshakeException.class, sslSocket::startHandshake); assertThrows(SSLHandshakeException.class, sslSocket::startHandshake);
} }
private void startSecureServer(final boolean setServerTrustStoreProperties) throws InitializationException { private void startSecureServer() {
final SSLContextService sslContextService = configureProcessorSslContextService(ListenHTTP.ClientAuthentication.AUTO, setServerTrustStoreProperties);
runner.setProperty(sslContextService, StandardSSLContextService.SSL_ALGORITHM, TlsConfiguration.TLS_PROTOCOL);
runner.enableControllerService(sslContextService);
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.RETURN_CODE, Integer.toString(HttpServletResponse.SC_NO_CONTENT)); runner.setProperty(ListenHTTP.RETURN_CODE, Integer.toString(HttpServletResponse.SC_NO_CONTENT));
@ -352,11 +389,10 @@ public class TestListenHTTP {
startWebServer(); startWebServer();
} }
private HttpsURLConnection getSecureConnection(final TlsConfiguration tlsConfiguration) throws Exception { private HttpsURLConnection getSecureConnection(final SSLContext sslContext) throws Exception {
final URL url = new URL(buildUrl(true)); final URL url = new URL(buildUrl(true));
final HttpsURLConnection connection = (HttpsURLConnection) url.openConnection(); final HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
final SSLSocketFactory sslSocketFactory = SslContextFactory.createSSLSocketFactory(tlsConfiguration); connection.setSSLSocketFactory(sslContext.getSocketFactory());
connection.setSSLSocketFactory(sslSocketFactory);
return connection; return connection;
} }
@ -383,18 +419,16 @@ public class TestListenHTTP {
return connection.getResponseCode(); return connection.getResponseCode();
} }
private static HttpsURLConnection buildSecureConnection(boolean twoWaySsl, URL url) throws IOException, TlsException { private static HttpsURLConnection buildSecureConnection(boolean twoWaySsl, URL url) throws IOException {
final HttpsURLConnection sslCon = (HttpsURLConnection) url.openConnection(); final HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
SSLContext clientSslContext;
if (twoWaySsl) { if (twoWaySsl) {
// Use a client certificate, do not reuse the server's keystore // Use a client certificate, do not reuse the server's keystore
clientSslContext = SslContextFactory.createSslContext(clientTlsConfiguration); connection.setSSLSocketFactory(keyStoreSslContext.getSocketFactory());
} else { } else {
// With one-way SSL, the client still needs a truststore // With one-way SSL, the client still needs a truststore
clientSslContext = SslContextFactory.createSslContext(trustOnlyTlsConfiguration); connection.setSSLSocketFactory(trustStoreSslContext.getSocketFactory());
} }
sslCon.setSSLSocketFactory(clientSslContext.getSocketFactory()); return connection;
return sslCon;
} }
private String buildUrl(final boolean secure) { private String buildUrl(final boolean secure) {
@ -423,21 +457,45 @@ public class TestListenHTTP {
final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory(); final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
final ProcessContext context = runner.getProcessContext(); final ProcessContext context = runner.getProcessContext();
proc.onTrigger(context, processSessionFactory); proc.onTrigger(context, processSessionFactory);
final int port = context.getProperty(ListenHTTP.PORT).evaluateAttributeExpressions().asInteger();
final InetSocketAddress socketAddress = new InetSocketAddress(LOCALHOST, port);
final Socket socket = new Socket();
boolean connected = false;
long elapsed = 0;
while (!connected && elapsed < SERVER_START_TIMEOUT) {
final long started = System.currentTimeMillis();
try {
socket.connect(socketAddress, SOCKET_CONNECT_TIMEOUT);
connected = true;
runner.getLogger().debug("Server Socket Connected after {} ms", new Object[]{elapsed});
socket.close();
} catch (final Exception e) {
runner.getLogger().debug("Server Socket Connect Failed: [{}] {}", new Object[]{e.getClass(), e.getMessage()});
}
final long connectElapsed = System.currentTimeMillis() - started;
elapsed += connectElapsed;
}
if (!connected) {
final String message = String.format("HTTP Server Port [%d] not listening after %d ms", port, SERVER_START_TIMEOUT);
throw new IllegalStateException(message);
}
} }
private void startWebServerAndSendRequests(Runnable sendRequestToWebserver, int numberOfExpectedFlowFiles) throws Exception { private void startWebServerAndSendRequests(Runnable sendRequestToWebserver, int numberOfExpectedFlowFiles) throws Exception {
startWebServer(); startWebServer();
new Thread(sendRequestToWebserver).start(); new Thread(sendRequestToWebserver).start();
long responseTimeout = 10000;
final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory(); final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
final ProcessContext context = runner.getProcessContext(); final ProcessContext context = runner.getProcessContext();
int numTransferred = 0; int numTransferred = 0;
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
while (numTransferred < numberOfExpectedFlowFiles && (System.currentTimeMillis() - startTime < responseTimeout)) { while (numTransferred < numberOfExpectedFlowFiles && (System.currentTimeMillis() - startTime < RESPONSE_TIMEOUT)) {
proc.onTrigger(context, processSessionFactory); proc.onTrigger(context, processSessionFactory);
numTransferred = runner.getFlowFilesForRelationship(RELATIONSHIP_SUCCESS).size(); numTransferred = runner.getFlowFilesForRelationship(RELATIONSHIP_SUCCESS).size();
Thread.sleep(100); Thread.sleep(SEND_REQUEST_SLEEP);
} }
runner.assertTransferCount(ListenHTTP.RELATIONSHIP_SUCCESS, numberOfExpectedFlowFiles); runner.assertTransferCount(ListenHTTP.RELATIONSHIP_SUCCESS, numberOfExpectedFlowFiles);
@ -462,33 +520,23 @@ public class TestListenHTTP {
startWebServerAndSendRequests(sendMessagesToWebServer, messages.size()); startWebServerAndSendRequests(sendMessagesToWebServer, messages.size());
} }
private SSLContextService configureProcessorSslContextService(boolean setTrustStoreProperties) throws InitializationException { private void configureProcessorSslContextService(final ListenHTTP.ClientAuthentication clientAuthentication,
ListenHTTP.ClientAuthentication clientAuthentication = ListenHTTP.ClientAuthentication.AUTO; final TlsConfiguration tlsConfiguration) throws InitializationException {
if (setTrustStoreProperties) { final RestrictedSSLContextService sslContextService = Mockito.mock(RestrictedSSLContextService.class);
clientAuthentication = ListenHTTP.ClientAuthentication.REQUIRED; Mockito.when(sslContextService.getIdentifier()).thenReturn(SSL_CONTEXT_SERVICE_IDENTIFIER);
} Mockito.when(sslContextService.createTlsConfiguration()).thenReturn(tlsConfiguration);
return configureProcessorSslContextService(clientAuthentication, setTrustStoreProperties);
}
private SSLContextService configureProcessorSslContextService(final ListenHTTP.ClientAuthentication clientAuthentication, if (ListenHTTP.ClientAuthentication.REQUIRED.equals(clientAuthentication)) {
final boolean setTrustStoreProperties) throws InitializationException { Mockito.when(sslContextService.createContext()).thenReturn(serverKeyStoreSslContext);
final SSLContextService sslContextService = new StandardRestrictedSSLContextService(); } else {
Mockito.when(sslContextService.createContext()).thenReturn(serverKeyStoreNoTrustStoreSslContext);
}
runner.addControllerService(SSL_CONTEXT_SERVICE_IDENTIFIER, sslContextService); runner.addControllerService(SSL_CONTEXT_SERVICE_IDENTIFIER, sslContextService);
if (setTrustStoreProperties) {
runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, TRUSTSTORE);
runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, TRUSTSTORE_PASSWORD);
runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, KeystoreType.JKS.getType());
}
runner.setProperty(ListenHTTP.CLIENT_AUTHENTICATION, clientAuthentication.name()); runner.setProperty(ListenHTTP.CLIENT_AUTHENTICATION, clientAuthentication.name());
runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE, KEYSTORE);
runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_PASSWORD, TRUSTSTORE_PASSWORD);
runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_TYPE, KeystoreType.JKS.getType());
runner.setProperty(ListenHTTP.SSL_CONTEXT_SERVICE, SSL_CONTEXT_SERVICE_IDENTIFIER); runner.setProperty(ListenHTTP.SSL_CONTEXT_SERVICE, SSL_CONTEXT_SERVICE_IDENTIFIER);
return sslContextService; runner.enableControllerService(sslContextService);
} }
private SSLContextService configureInvalidProcessorSslContextService() throws InitializationException { private SSLContextService configureInvalidProcessorSslContextService() throws InitializationException {

3
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutTCP.java
View File

@ -22,7 +22,6 @@ public class TestPutTCP extends TestPutTCPCommon {
public TestPutTCP() { public TestPutTCP() {
super(); super();
ssl = false;
} }
@Override @Override
@ -41,6 +40,4 @@ public class TestPutTCP extends TestPutTCPCommon {
runner.assertNotValid(); runner.assertNotValid();
} }
} }
} }

61
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutTcpSSL.java
View File

@ -16,23 +16,42 @@
*/ */
package org.apache.nifi.processors.standard; package org.apache.nifi.processors.standard;
import java.util.HashMap;
import java.util.Map;
import org.apache.nifi.processors.standard.util.TestPutTCPCommon; import org.apache.nifi.processors.standard.util.TestPutTCPCommon;
import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.security.util.KeystoreType;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.ssl.SSLContextService;
import org.junit.BeforeClass;
import org.mockito.Mockito;
import javax.net.ssl.SSLContext;
public class TestPutTcpSSL extends TestPutTCPCommon { public class TestPutTcpSSL extends TestPutTCPCommon {
private static Map<String, String> sslProperties; private static final String TLS_PROTOCOL = "TLSv1.2";
// TODO: The NiFi SSL classes don't yet support TLSv1.3, so set the CS version explicitly private static SSLContext sslContext;
private static final String TLS_PROTOCOL_VERSION = "TLSv1.2";
@BeforeClass
public static void configureServices() throws TlsException {
final TlsConfiguration configuration = new StandardTlsConfiguration(
"src/test/resources/keystore.jks",
"passwordpassword",
"passwordpassword",
KeystoreType.JKS,
"src/test/resources/truststore.jks",
"passwordpassword",
KeystoreType.JKS,
TLS_PROTOCOL
);
sslContext = SslContextFactory.createSslContext(configuration);
}
public TestPutTcpSSL() { public TestPutTcpSSL() {
super(); super();
ssl = true; serverSocketFactory = sslContext.getServerSocketFactory();
sslProperties = createSslProperties();
} }
@Override @Override
@ -40,10 +59,14 @@ public class TestPutTcpSSL extends TestPutTCPCommon {
runner.setProperty(PutTCP.HOSTNAME, host); runner.setProperty(PutTCP.HOSTNAME, host);
runner.setProperty(PutTCP.PORT, Integer.toString(port)); runner.setProperty(PutTCP.PORT, Integer.toString(port));
final StandardSSLContextService sslService = new StandardSSLContextService(); final SSLContextService sslContextService = Mockito.mock(SSLContextService.class);
runner.addControllerService("ssl-context", sslService, sslProperties); final String serviceIdentifier = SSLContextService.class.getName();
runner.enableControllerService(sslService); Mockito.when(sslContextService.getIdentifier()).thenReturn(serviceIdentifier);
runner.setProperty(PutTCP.SSL_CONTEXT_SERVICE, "ssl-context"); Mockito.when(sslContextService.createContext()).thenReturn(sslContext);
runner.addControllerService(serviceIdentifier, sslContextService);
runner.enableControllerService(sslContextService);
runner.setProperty(PutTCP.SSL_CONTEXT_SERVICE, serviceIdentifier);
if (outgoingMessageDelimiter != null) { if (outgoingMessageDelimiter != null) {
runner.setProperty(PutTCP.OUTGOING_MESSAGE_DELIMITER, outgoingMessageDelimiter); runner.setProperty(PutTCP.OUTGOING_MESSAGE_DELIMITER, outgoingMessageDelimiter);
@ -56,16 +79,4 @@ public class TestPutTcpSSL extends TestPutTCPCommon {
runner.assertNotValid(); runner.assertNotValid();
} }
} }
private static Map<String, String> createSslProperties() {
final Map<String, String> map = new HashMap<>();
map.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/keystore.jks");
map.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "passwordpassword");
map.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
map.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/truststore.jks");
map.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "passwordpassword");
map.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
map.put(StandardSSLContextService.SSL_ALGORITHM.getName(), TLS_PROTOCOL_VERSION);
return map;
}
} }

17
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TCPTestServer.java
View File

@ -25,10 +25,6 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
import javax.net.ServerSocketFactory; import javax.net.ServerSocketFactory;
import javax.net.ssl.SSLContext;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TlsConfiguration;
public class TCPTestServer implements Runnable { public class TCPTestServer implements Runnable {
@ -51,17 +47,12 @@ public class TCPTestServer implements Runnable {
this.messageDelimiter = messageDelimiter; this.messageDelimiter = messageDelimiter;
} }
public synchronized void startServer(boolean ssl) throws Exception { public synchronized void startServer(final ServerSocketFactory serverSocketFactory) throws Exception {
if (!isServerRunning()) { if (!isServerRunning()) {
if(ssl){ if (serverSocketFactory == null) {
TlsConfiguration tlsConfiguration = new StandardTlsConfiguration("src/test/resources/keystore.jks","passwordpassword", null, "JKS", "src/test/resources/truststore.jks",
"passwordpassword", "JKS", TlsConfiguration.getHighestCurrentSupportedTlsProtocolVersion());
final SSLContext sslCtx = SslContextFactory.createSslContext(tlsConfiguration);
ServerSocketFactory sslSocketFactory = sslCtx.getServerSocketFactory();
serverSocket = sslSocketFactory.createServerSocket(0, 0, ipAddress);
} else {
serverSocket = new ServerSocket(0, 0, ipAddress); serverSocket = new ServerSocket(0, 0, ipAddress);
} else {
serverSocket = serverSocketFactory.createServerSocket(0, 0, ipAddress);
} }
Thread t = new Thread(this); Thread t = new Thread(this);
t.setName(this.getClass().getSimpleName()); t.setName(this.getClass().getSimpleName());

67
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java
View File

@ -33,6 +33,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import javax.net.ssl.SSLContext;
import javax.servlet.MultipartConfigElement; import javax.servlet.MultipartConfigElement;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -45,10 +46,12 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processors.standard.InvokeHTTP; import org.apache.nifi.processors.standard.InvokeHTTP;
import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType; import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.remote.io.socket.NetworkUtils;
import org.apache.nifi.security.util.ClientAuth;
import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.StringUtils; import org.apache.nifi.util.StringUtils;
import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunner;
import org.apache.nifi.web.util.TestServer; import org.apache.nifi.web.util.JettyServerUtils;
import org.eclipse.jetty.http.MultiPartFormInputStream; import org.eclipse.jetty.http.MultiPartFormInputStream;
import org.eclipse.jetty.security.ConstraintSecurityHandler; import org.eclipse.jetty.security.ConstraintSecurityHandler;
import org.eclipse.jetty.security.DefaultIdentityService; import org.eclipse.jetty.security.DefaultIdentityService;
@ -58,20 +61,63 @@ import org.eclipse.jetty.security.authentication.DigestAuthenticator;
import org.eclipse.jetty.server.Authentication; import org.eclipse.jetty.server.Authentication;
import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.server.handler.AbstractHandler;
import org.eclipse.jetty.util.MultiPartInputStreamParser; import org.eclipse.jetty.util.MultiPartInputStreamParser;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
public abstract class TestInvokeHttpCommon { public abstract class TestInvokeHttpCommon {
public static TestServer server; protected static Server server;
public static String url;
public TestRunner runner; protected static String url;
public void addHandler(Handler handler) { private static final long READ_TIMEOUT_SLEEP = 1000;
server.addHandler(handler);
protected TestRunner runner;
protected static void configureServer(final SSLContext sslContext, final ClientAuth clientAuth) throws Exception {
final int port = NetworkUtils.availablePort();
final String protocol = sslContext == null ? "http" : "https";
setUrl(protocol, port);
final Server configuredServer = JettyServerUtils.createServer(port, sslContext, clientAuth);
final ServerConnector connector = new ServerConnector(configuredServer);
connector.setPort(port);
JettyServerUtils.startServer(configuredServer);
setServer(configuredServer);
}
private static void setUrl(final String scheme, final int port) {
url = String.format("%s://localhost:%d", scheme, port);
}
private static void setServer(final Server configuredServer) {
server = configuredServer;
}
protected static void addHandler(final Handler handler) {
JettyServerUtils.addHandler(server, handler);
}
@AfterClass
public static void afterClass() throws Exception {
if (server != null) {
server.stop();
server.destroy();
}
}
@After
public void after() {
JettyServerUtils.clearHandlers(server);
runner.shutdown();
} }
@Test @Test
@ -84,6 +130,9 @@ public abstract class TestInvokeHttpCommon {
createFlowFiles(runner); createFlowFiles(runner);
runner.run(); runner.run();
runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
// extract the date string sent to the server // extract the date string sent to the server
// and store it as a java.util.Date // and store it as a java.util.Date
final SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); final SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US);
@ -100,7 +149,6 @@ public abstract class TestInvokeHttpCommon {
if (diff > threshold) { if (diff > threshold) {
fail("Difference (" + diff + ") was greater than threshold (" + threshold + ")"); fail("Difference (" + diff + ") was greater than threshold (" + threshold + ")");
} }
System.out.println("diff: " + diff);
} }
@Test @Test
@ -773,7 +821,6 @@ public abstract class TestInvokeHttpCommon {
createFlowFiles(runner); createFlowFiles(runner);
//assertTrue(server.jetty.isRunning());
runner.run(); runner.run();
runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0); runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0); runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
@ -1489,7 +1536,7 @@ public abstract class TestInvokeHttpCommon {
addHandler(new ReadTimeoutHandler()); addHandler(new ReadTimeoutHandler());
runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200"); runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
runner.setProperty(InvokeHTTP.PROP_READ_TIMEOUT, "5 secs"); runner.setProperty(InvokeHTTP.PROP_READ_TIMEOUT, String.format("%d ms", READ_TIMEOUT_SLEEP / 2));
createFlowFiles(runner); createFlowFiles(runner);
@ -2020,7 +2067,7 @@ public abstract class TestInvokeHttpCommon {
if ("Get".equalsIgnoreCase(request.getMethod())) { if ("Get".equalsIgnoreCase(request.getMethod())) {
try { try {
Thread.sleep(10000); Thread.sleep(READ_TIMEOUT_SLEEP);
} catch (InterruptedException e) { } catch (InterruptedException e) {
return; return;
} }

11
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestPutTCPCommon.java
View File

@ -30,6 +30,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import javax.net.ServerSocketFactory;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.List; import java.util.List;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
@ -59,7 +60,7 @@ public abstract class TestPutTCPCommon {
private final static char CONTENT_CHAR = 'x'; private final static char CONTENT_CHAR = 'x';
private final static int DATA_WAIT_PERIOD = 1000; private final static int DATA_WAIT_PERIOD = 1000;
private final static int DEFAULT_TEST_TIMEOUT_PERIOD = 10000; private final static int DEFAULT_TEST_TIMEOUT_PERIOD = 10000;
private final static int LONG_TEST_TIMEOUT_PERIOD = 100000; private final static int LONG_TEST_TIMEOUT_PERIOD = 180000;
private final static String OUTGOING_MESSAGE_DELIMITER = "\n"; private final static String OUTGOING_MESSAGE_DELIMITER = "\n";
private final static String OUTGOING_MESSAGE_DELIMITER_MULTI_CHAR = "{delimiter}\r\n"; private final static String OUTGOING_MESSAGE_DELIMITER_MULTI_CHAR = "{delimiter}\r\n";
@ -67,7 +68,7 @@ public abstract class TestPutTCPCommon {
private int tcp_server_port; private int tcp_server_port;
private ArrayBlockingQueue<List<Byte>> recvQueue; private ArrayBlockingQueue<List<Byte>> recvQueue;
public boolean ssl; public ServerSocketFactory serverSocketFactory;
public TestRunner runner; public TestRunner runner;
// Test Data // Test Data
@ -88,7 +89,7 @@ public abstract class TestPutTCPCommon {
private synchronized TCPTestServer createTestServer(final String address, final ArrayBlockingQueue<List<Byte>> recvQueue, final String delimiter) throws Exception { private synchronized TCPTestServer createTestServer(final String address, final ArrayBlockingQueue<List<Byte>> recvQueue, final String delimiter) throws Exception {
TCPTestServer server = new TCPTestServer(InetAddress.getByName(address), recvQueue, delimiter); TCPTestServer server = new TCPTestServer(InetAddress.getByName(address), recvQueue, delimiter);
server.startServer(ssl); server.startServer(serverSocketFactory);
tcp_server_port = server.getPort(); tcp_server_port = server.getPort();
return server; return server;
} }
@ -136,8 +137,8 @@ public abstract class TestPutTCPCommon {
checkReceivedAllData(recvQueue, VALID_FILES); checkReceivedAllData(recvQueue, VALID_FILES);
checkInputQueueIsEmpty(); checkInputQueueIsEmpty();
checkTotalNumConnections(server, 1); checkTotalNumConnections(server, 1);
runner.setProperty(PutTCP.IDLE_EXPIRATION, "1 second"); runner.setProperty(PutTCP.IDLE_EXPIRATION, "100 ms");
Thread.sleep(2000); Thread.sleep(200);
runner.run(1, false, false); runner.run(1, false, false);
runner.clearTransferState(); runner.clearTransferState();
sendTestData(VALID_FILES); sendTestData(VALID_FILES);

88
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/JettyServerUtils.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.web.util;
import org.apache.nifi.security.util.ClientAuth;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import javax.net.ssl.SSLContext;
public class JettyServerUtils {
private static final long IDLE_TIMEOUT = 60000;
private static final long SERVER_START_SLEEP = 100L;
public static Server createServer(final int port, final SSLContext sslContext, final ClientAuth clientAuth) {
final Server server = new Server();
final ServerConnector connector;
if (sslContext == null) {
connector = new ServerConnector(server);
} else {
final SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
if (ClientAuth.REQUIRED.equals(clientAuth)) {
sslContextFactory.setNeedClientAuth(true);
}
connector = new ServerConnector(server, sslContextFactory);
}
connector.setPort(port);
connector.setIdleTimeout(IDLE_TIMEOUT);
server.addConnector(connector);
final HandlerCollection handlerCollection = new HandlerCollection(true);
server.setHandler(handlerCollection);
return server;
}
public static void startServer(final Server server) throws Exception {
server.start();
while (!server.isStarted()) {
Thread.sleep(SERVER_START_SLEEP);
}
}
public static void addHandler(final Server server, final Handler handler) {
final Handler serverHandler = server.getHandler();
if (serverHandler instanceof HandlerCollection) {
final HandlerCollection handlerCollection = (HandlerCollection) serverHandler;
handlerCollection.addHandler(handler);
}
}
public static void clearHandlers(final Server server) {
final Handler serverHandler = server.getHandler();
if (serverHandler instanceof HandlerCollection) {
final HandlerCollection handlerCollection = (HandlerCollection) serverHandler;
final Handler[] handlers = handlerCollection.getHandlers();
if (handlers != null) {
for (final Handler handler : handlerCollection.getHandlers()) {
handlerCollection.removeHandler(handler);
}
}
}
}
}

20
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java
View File

@ -31,6 +31,8 @@ public class TestServer {
public static final String NEED_CLIENT_AUTH = "clientAuth"; public static final String NEED_CLIENT_AUTH = "clientAuth";
private static final long IDLE_TIMEOUT = 60000;
private Server jetty; private Server jetty;
private boolean secure = false; private boolean secure = false;
@ -71,12 +73,12 @@ public class TestServer {
final ServerConnector http = new ServerConnector(jetty); final ServerConnector http = new ServerConnector(jetty);
http.setPort(0); http.setPort(0);
// Severely taxed environments may have significant delays when executing. // Severely taxed environments may have significant delays when executing.
http.setIdleTimeout(30000L); http.setIdleTimeout(IDLE_TIMEOUT);
jetty.addConnector(http); jetty.addConnector(http);
} }
private void createSecureConnector(final Map<String, String> sslProperties) { private void createSecureConnector(final Map<String, String> sslProperties) {
SslContextFactory ssl = new SslContextFactory.Server(); SslContextFactory.Server ssl = new SslContextFactory.Server();
if (sslProperties.get(StandardSSLContextService.KEYSTORE.getName()) != null) { if (sslProperties.get(StandardSSLContextService.KEYSTORE.getName()) != null) {
ssl.setKeyStorePath(sslProperties.get(StandardSSLContextService.KEYSTORE.getName())); ssl.setKeyStorePath(sslProperties.get(StandardSSLContextService.KEYSTORE.getName()));
@ -103,7 +105,7 @@ public class TestServer {
// set host and port // set host and port
https.setPort(0); https.setPort(0);
// Severely taxed environments may have significant delays when executing. // Severely taxed environments may have significant delays when executing.
https.setIdleTimeout(30000L); https.setIdleTimeout(IDLE_TIMEOUT);
// add the connector // add the connector
jetty.addConnector(https); jetty.addConnector(https);
@ -113,17 +115,11 @@ public class TestServer {
} }
public void clearHandlers() { public void clearHandlers() {
HandlerCollection hc = (HandlerCollection) jetty.getHandler(); JettyServerUtils.clearHandlers(jetty);
Handler[] ha = hc.getHandlers();
if (ha != null) {
for (Handler h : ha) {
hc.removeHandler(h);
}
}
} }
public void addHandler(Handler handler) { public void addHandler(final Handler handler) {
((HandlerCollection) jetty.getHandler()).addHandler(handler); JettyServerUtils.addHandler(jetty, handler);
} }
public void startServer() throws Exception { public void startServer() throws Exception {

15
nifi-nar-bundles/nifi-standard-bundle/pom.xml
View File

@ -181,21 +181,6 @@
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.9</version> <version>3.9</version>
</dependency> </dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpg-jdk15on</artifactId>
<version>1.66</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.66</version>
</dependency>
<dependency> <dependency>
<groupId>commons-codec</groupId> <groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId> <artifactId>commons-codec</artifactId>

2
nifi-toolkit/nifi-toolkit-tls/pom.xml
View File

@ -51,12 +51,10 @@
<dependency> <dependency>
<groupId>org.bouncycastle</groupId> <groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId> <artifactId>bcpkix-jdk15on</artifactId>
<version>1.66</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.bouncycastle</groupId> <groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId> <artifactId>bcprov-jdk15on</artifactId>
<version>1.66</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-cli</groupId> <groupId>commons-cli</groupId>

16
pom.xml
View File

@ -88,6 +88,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<inceptionYear>2014</inceptionYear> <inceptionYear>2014</inceptionYear>
<org.bouncycastle.version>1.68</org.bouncycastle.version>
<org.slf4j.version>1.7.30</org.slf4j.version> <org.slf4j.version>1.7.30</org.slf4j.version>
<ranger.version>2.1.0</ranger.version> <ranger.version>2.1.0</ranger.version>
<jetty.version>9.4.35.v20201120</jetty.version> <jetty.version>9.4.35.v20201120</jetty.version>
@ -261,6 +262,21 @@
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>
<version>${org.slf4j.version}</version> <version>${org.slf4j.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>${org.bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>${org.bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpg-jdk15on</artifactId>
<version>${org.bouncycastle.version}</version>
</dependency>
<!-- These junit/mockito/groovy/spock/hamcrest dependencies are here to encourage consistent unit test library usage --> <!-- These junit/mockito/groovy/spock/hamcrest dependencies are here to encourage consistent unit test library usage -->
<dependency> <dependency>