Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-7294 Address deprecation issues in solrj and httpclient 

Some calls to deprecated methods in httpclient were resulting in
UnsupportedOperationException. Use the new API calls in both httpclient
and solrj. Add an integration test to include test coverage for
org.apache.nifi.processors.solr.SolrUtils.createClient

This closes #4171.
This commit is contained in:
Mubashir Kazia 2020-03-30 14:20:05 -04:00 committed by Bryan Bende
parent 992d990dd2
commit fee1b8b8e0
No known key found for this signature in database
GPG Key ID: A0DDA9ED50711C39
6 changed files with 121 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java
View File

@ -21,8 +21,11 @@ package org.apache.nifi.processors.solr;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
@ -230,7 +233,7 @@ public class SolrUtils {
public static final String REPEATING_PARAM_PATTERN = "[\\w\\.]+\\.\\d+$";
public static SolrClient createSolrClient(final PropertyContext context, final String solrLocation) {
public static synchronized SolrClient createSolrClient(final PropertyContext context, final String solrLocation) {
final Integer socketTimeout = context.getProperty(SOLR_SOCKET_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger();
@ -240,26 +243,36 @@ public class SolrUtils {
final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).evaluateAttributeExpressions().getValue();
final String kerberosPassword = context.getProperty(KERBEROS_PASSWORD).getValue();
final ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
// Reset HttpClientBuilder static values
HttpClientUtil.resetHttpClientBuilder();
// has to happen before the client is created below so that correct configurer would be set if needed
if (kerberosCredentialsService != null || (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword))) {
HttpClientUtil.setHttpClientBuilder(new KerberosHttpClientBuilder().getHttpClientBuilder(Optional.empty()));
}
final HttpClient httpClient = HttpClientUtil.createClient(params);
if (sslContextService != null) {
final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
final SSLSocketFactory sslSocketFactory = new SSLSocketFactory(sslContext);
final Scheme httpsScheme = new Scheme("https", 443, sslSocketFactory);
httpClient.getConnectionManager().getSchemeRegistry().register(httpsScheme);
final SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext);
HttpClientUtil.setSchemaRegistryProvider(new HttpClientUtil.SchemaRegistryProvider() {
@Override
public Registry<ConnectionSocketFactory> getSchemaRegistry() {
RegistryBuilder<ConnectionSocketFactory> builder = RegistryBuilder.create();
builder.register("http", PlainConnectionSocketFactory.getSocketFactory());
builder.register("https", sslSocketFactory);
return builder.build();
}
});
}
final ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
final HttpClient httpClient = HttpClientUtil.createClient(params);
if (SOLR_TYPE_STANDARD.getValue().equals(context.getProperty(SOLR_TYPE).getValue())) {
return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build();
} else {

92
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java
View File

@ -20,10 +20,13 @@
package org.apache.nifi.processors.solr;
import com.google.gson.stream.JsonReader;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.json.JsonRecordSetWriter;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.schema.access.SchemaAccessUtils;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
@ -36,8 +39,10 @@ import org.apache.solr.common.SolrInputDocument;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
import org.xmlunit.matchers.CompareMatcher;
import javax.net.ssl.SSLContext;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
@ -45,16 +50,18 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.TimeZone;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
public class QuerySolrIT {
/*
@ -117,7 +124,7 @@ public class QuerySolrIT {
CloudSolrClient solrClient = null;
try {
solrClient = new CloudSolrClient.Builder().withZkHost(SOLR_LOCATION).build();
solrClient = new CloudSolrClient.Builder(Collections.singletonList(SOLR_LOCATION), Optional.empty()).build();
solrClient.setDefaultCollection(SOLR_COLLECTION);
} catch (Exception e) {
e.printStackTrace();
@ -141,7 +148,7 @@ public class QuerySolrIT {
TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(SolrUtils.SOLR_TYPE, SolrUtils.SOLR_TYPE_CLOUD.getValue());
runner.setProperty(SolrUtils.SOLR_LOCATION, "localhost:2181");
runner.setProperty(SolrUtils.SOLR_LOCATION, SOLR_LOCATION);
runner.setProperty(SolrUtils.COLLECTION, SOLR_COLLECTION);
return runner;
@ -627,6 +634,24 @@ public class QuerySolrIT {
assertEquals(controlScore, 45);
}
@Test
public void testSslContextService() throws IOException, InitializationException {
final QuerySolr proc = Mockito.mock(QuerySolr.class);
TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(SolrUtils.SOLR_TYPE, SolrUtils.SOLR_TYPE_CLOUD.getValue());
runner.setProperty(SolrUtils.SOLR_LOCATION, SOLR_LOCATION);
runner.setProperty(SolrUtils.COLLECTION, SOLR_COLLECTION);
final SSLContextService sslContextService = new MockSSLContextService();
runner.addControllerService("ssl-context", sslContextService);
runner.enableControllerService(sslContextService);
runner.setProperty(SolrUtils.SSL_CONTEXT_SERVICE, "ssl-context");
proc.onScheduled(runner.getProcessContext());
Mockito.verify(proc, Mockito.times(1)).createSolrClient(Mockito.any(ProcessContext.class), Mockito.eq((String)SOLR_LOCATION));
}
// Override createSolrClient and return the passed in SolrClient
private class TestableProcessor extends QuerySolr {
private SolrClient solrClient;
@ -639,4 +664,65 @@ public class QuerySolrIT {
return solrClient;
}
}
/**
* Mock implementation so we don't need to have a real keystore/truststore available for testing.
*/
private class MockSSLContextService extends AbstractControllerService implements SSLContextService {
@Override
public SSLContext createSSLContext(ClientAuth clientAuth) throws ProcessException {
return null;
}
@Override
public String getTrustStoreFile() {
return null;
}
@Override
public String getTrustStoreType() {
return null;
}
@Override
public String getTrustStorePassword() {
return null;
}
@Override
public boolean isTrustStoreConfigured() {
return false;
}
@Override
public String getKeyStoreFile() {
return null;
}
@Override
public String getKeyStoreType() {
return null;
}
@Override
public String getKeyStorePassword() {
return null;
}
@Override
public String getKeyPassword() {
return null;
}
@Override
public boolean isKeyStoreConfigured() {
return false;
}
@Override
public String getSslAlgorithm() {
return null;
}
}
}

2
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java
View File

@ -48,8 +48,8 @@ import java.util.Date;
import java.util.Locale;
import java.util.TimeZone;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
public class TestGetSolr {

4
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestPutSolrContentStream.java
View File

@ -30,7 +30,7 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
@ -318,7 +318,7 @@ public class TestPutSolrContentStream {
@Test
public void testRemoteSolrExceptionShouldRouteToFailure() throws IOException, SolrServerException {
final Throwable throwable = new HttpSolrClient.RemoteSolrException(
final Throwable throwable = new BaseHttpSolrClient.RemoteSolrException(
"host", 401, "error", new NumberFormatException());
final ExceptionThrowingProcessor proc = new ExceptionThrowingProcessor(throwable);

4
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestPutSolrRecord.java
View File

@ -36,7 +36,7 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
@ -440,7 +440,7 @@ public class TestPutSolrRecord {
@Test
public void testRemoteSolrExceptionShouldRouteToFailure() throws IOException, SolrServerException, InitializationException {
final Throwable throwable = new HttpSolrClient.RemoteSolrException(
final Throwable throwable = new BaseHttpSolrClient.RemoteSolrException(
"host", 401, "error", new NumberFormatException());
final ExceptionThrowingProcessor proc = new ExceptionThrowingProcessor(throwable);

2
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java
View File

@ -48,9 +48,9 @@ import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
public class TestQuerySolr {
static final String DEFAULT_SOLR_CORE = "testCollection";