NIFI-1768 Adding TLS/SSL support to Solr processors NIFI-1980 Added a default value for PutSolrContentStream commitWithIn NIFI-2568 Added Kerberos support to Solr processors

Upgrading SolrJ to 6.2.

Signed-off-by: Yolanda M. Davis <ymdavis@apache.org>

This closes #1005
This commit is contained in:
Bryan Bende 2016-09-07 22:11:10 -04:00 committed by Yolanda M. Davis
parent c4826cac54
commit d4948a3778
19 changed files with 293 additions and 31 deletions

View File

@ -35,6 +35,11 @@
<artifactId>nifi-solr-processors</artifactId> <artifactId>nifi-solr-processors</artifactId>
<version>1.1.0-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-standard-services-api-nar</artifactId>
<type>nar</type>
</dependency>
</dependencies> </dependencies>
</project> </project>

View File

@ -37,6 +37,23 @@ The following binary components are provided under the Apache Software License v
This product includes software from the Spring Framework, This product includes software from the Spring Framework,
under the Apache License 2.0 (see: StringUtils.containsWhitespace()) under the Apache License 2.0 (see: StringUtils.containsWhitespace())
(ASLv2) Apache Commons Codec
The following NOTICE information applies:
Apache Commons Codec
Copyright 2002-2014 The Apache Software Foundation
src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
contains test data from http://aspell.net/test/orig/batch0.tab.
Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
===============================================================================
The content of package org.apache.commons.codec.language.bm has been translated
from the original php source code available at http://stevemorse.org/phoneticinfo.htm
with permission from the original authors.
Original source copyright:
Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
(ASLv2) Apache ZooKeeper (ASLv2) Apache ZooKeeper
The following NOTICE information applies: The following NOTICE information applies:
Apache ZooKeeper Apache ZooKeeper

View File

@ -31,6 +31,11 @@
<artifactId>solr-solrj</artifactId> <artifactId>solr-solrj</artifactId>
<version>${solr.version}</version> <version>${solr.version}</version>
</dependency> </dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.10</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.nifi</groupId> <groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId> <artifactId>nifi-api</artifactId>
@ -39,6 +44,11 @@
<groupId>org.apache.nifi</groupId> <groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId> <artifactId>nifi-processor-utils</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-ssl-context-service-api</artifactId>
<scope>provided</scope>
</dependency>
<!-- test dependencies --> <!-- test dependencies -->
<dependency> <dependency>
<groupId>org.apache.nifi</groupId> <groupId>org.apache.nifi</groupId>
@ -68,8 +78,7 @@
<version>${solr.version}</version> <version>${solr.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- These Lucene deps should be brought in through solr-core, but there <!-- Need to declare the newer versions of these b/c NiFi uses Lucene 4.10.3 -->
appears to be an issue with 5.0.0 that still references some 4.10.3 poms -->
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId> <artifactId>lucene-core</artifactId>
@ -97,18 +106,19 @@
<configuration> <configuration>
<excludes combine.children="append"> <excludes combine.children="append">
<exclude>src/test/resources/solr/solr.xml</exclude> <exclude>src/test/resources/solr/solr.xml</exclude>
<exclude>src/test/resources/testCollection/core.properties</exclude> <exclude>src/test/resources/solr/testCollection/core.properties</exclude>
<exclude>src/test/resources/testCollection/conf/_rest_managed.json</exclude> <exclude>src/test/resources/solr/testCollection/conf/_rest_managed.json</exclude>
<exclude>src/test/resources/testCollection/conf/protowords.txt</exclude> <exclude>src/test/resources/solr/testCollection/conf/protowords.txt</exclude>
<exclude>src/test/resources/testCollection/conf/schema.xml</exclude> <exclude>src/test/resources/solr/testCollection/conf/schema.xml</exclude>
<exclude>src/test/resources/testCollection/conf/solrconfig.xml</exclude> <exclude>src/test/resources/solr/testCollection/conf/solrconfig.xml</exclude>
<exclude>src/test/resources/testCollection/conf/synonyms.txt</exclude> <exclude>src/test/resources/solr/testCollection/conf/synonyms.txt</exclude>
<exclude>src/test/resources/testCollection/conf/lang/stopwords_en.txt</exclude> <exclude>src/test/resources/solr/testCollection/conf/lang/stopwords_en.txt</exclude>
<exclude>src/test/resources/testdata/test-csv-multiple-docs.csv</exclude> <exclude>src/test/resources/testdata/test-csv-multiple-docs.csv</exclude>
<exclude>src/test/resources/testdata/test-custom-json-single-doc.json</exclude> <exclude>src/test/resources/testdata/test-custom-json-single-doc.json</exclude>
<exclude>src/test/resources/testdata/test-solr-json-multiple-docs.json</exclude> <exclude>src/test/resources/testdata/test-solr-json-multiple-docs.json</exclude>
<exclude>src/test/resources/testdata/test-xml-multiple-docs.xml</exclude> <exclude>src/test/resources/testdata/test-xml-multiple-docs.xml</exclude>
<exclude>src/test/resources/log4j.properties</exclude> <exclude>src/test/resources/log4j.properties</exclude>
<exclude>src/test/resources/jaas-client.conf</exclude>
</excludes> </excludes>
</configuration> </configuration>
</plugin> </plugin>

View File

@ -23,6 +23,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -47,6 +48,7 @@ import org.apache.nifi.annotation.lifecycle.OnRemoved;
import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessSession;
@ -62,6 +64,7 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
@Tags({"Apache", "Solr", "Get", "Pull"}) @Tags({"Apache", "Solr", "Get", "Pull"})
@InputRequirement(Requirement.INPUT_FORBIDDEN) @InputRequirement(Requirement.INPUT_FORBIDDEN)
@ -139,6 +142,8 @@ public class GetSolr extends SolrProcessor {
descriptors.add(SORT_CLAUSE); descriptors.add(SORT_CLAUSE);
descriptors.add(DATE_FIELD); descriptors.add(DATE_FIELD);
descriptors.add(BATCH_SIZE); descriptors.add(BATCH_SIZE);
descriptors.add(JAAS_CLIENT_APP_NAME);
descriptors.add(SSL_CONTEXT_SERVICE);
descriptors.add(SOLR_SOCKET_TIMEOUT); descriptors.add(SOLR_SOCKET_TIMEOUT);
descriptors.add(SOLR_CONNECTION_TIMEOUT); descriptors.add(SOLR_CONNECTION_TIMEOUT);
descriptors.add(SOLR_MAX_CONNECTIONS); descriptors.add(SOLR_MAX_CONNECTIONS);
@ -235,6 +240,7 @@ public class GetSolr extends SolrProcessor {
if (documentList != null && documentList.getNumFound() > 0) { if (documentList != null && documentList.getNumFound() > 0) {
FlowFile flowFile = session.create(); FlowFile flowFile = session.create();
flowFile = session.write(flowFile, new QueryResponseOutputStreamCallback(response)); flowFile = session.write(flowFile, new QueryResponseOutputStreamCallback(response));
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/xml");
session.transfer(flowFile, REL_SUCCESS); session.transfer(flowFile, REL_SUCCESS);
StringBuilder transitUri = new StringBuilder("solr://"); StringBuilder transitUri = new StringBuilder("solr://");
@ -324,9 +330,19 @@ public class GetSolr extends SolrProcessor {
@Override @Override
public void process(OutputStream out) throws IOException { public void process(OutputStream out) throws IOException {
for (SolrDocument doc : response.getResults()) { for (SolrDocument doc : response.getResults()) {
String xml = ClientUtils.toXML(ClientUtils.toSolrInputDocument(doc)); String xml = ClientUtils.toXML(toSolrInputDocument(doc));
IOUtils.write(xml, out); IOUtils.write(xml, out, StandardCharsets.UTF_8);
} }
} }
public SolrInputDocument toSolrInputDocument(SolrDocument d) {
SolrInputDocument doc = new SolrInputDocument();
for (String name : d.getFieldNames()) {
doc.addField(name, d.getFieldValue(name));
}
return doc;
}
} }
} }

View File

@ -88,6 +88,7 @@ public class PutSolrContentStream extends SolrProcessor {
.required(false) .required(false)
.addValidator(StandardValidators.POSITIVE_LONG_VALIDATOR) .addValidator(StandardValidators.POSITIVE_LONG_VALIDATOR)
.expressionLanguageSupported(true) .expressionLanguageSupported(true)
.defaultValue("5000")
.build(); .build();
public static final Relationship REL_SUCCESS = new Relationship.Builder() public static final Relationship REL_SUCCESS = new Relationship.Builder()
@ -123,6 +124,8 @@ public class PutSolrContentStream extends SolrProcessor {
descriptors.add(CONTENT_STREAM_PATH); descriptors.add(CONTENT_STREAM_PATH);
descriptors.add(CONTENT_TYPE); descriptors.add(CONTENT_TYPE);
descriptors.add(COMMIT_WITHIN); descriptors.add(COMMIT_WITHIN);
descriptors.add(JAAS_CLIENT_APP_NAME);
descriptors.add(SSL_CONTEXT_SERVICE);
descriptors.add(SOLR_SOCKET_TIMEOUT); descriptors.add(SOLR_SOCKET_TIMEOUT);
descriptors.add(SOLR_CONNECTION_TIMEOUT); descriptors.add(SOLR_CONNECTION_TIMEOUT);
descriptors.add(SOLR_MAX_CONNECTIONS); descriptors.add(SOLR_MAX_CONNECTIONS);

View File

@ -18,7 +18,10 @@
*/ */
package org.apache.nifi.processors.solr; package org.apache.nifi.processors.solr;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.AllowableValue; import org.apache.nifi.components.AllowableValue;
@ -28,12 +31,16 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
import javax.net.ssl.SSLContext;
import javax.security.auth.login.Configuration;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
@ -76,6 +83,22 @@ public abstract class SolrProcessor extends AbstractProcessor {
.expressionLanguageSupported(true) .expressionLanguageSupported(true)
.build(); .build();
public static final PropertyDescriptor JAAS_CLIENT_APP_NAME = new PropertyDescriptor
.Builder().name("JAAS Client App Name")
.description("The name of the JAAS configuration entry to use when performing Kerberos authentication to Solr. If this property is " +
"not provided, Kerberos authentication will not be attempted. The value must match an entry in the file specified by the " +
"system property java.security.auth.login.config.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
.name("SSL Context Service")
.description("The Controller Service to use in order to obtain an SSL Context. This property must be set when communicating with a Solr over https.")
.required(false)
.identifiesControllerService(SSLContextService.class)
.build();
public static final PropertyDescriptor SOLR_SOCKET_TIMEOUT = new PropertyDescriptor public static final PropertyDescriptor SOLR_SOCKET_TIMEOUT = new PropertyDescriptor
.Builder().name("Solr Socket Timeout") .Builder().name("Solr Socket Timeout")
.description("The amount of time to wait for data on a socket connection to Solr. A value of 0 indicates an infinite timeout.") .description("The amount of time to wait for data on a socket connection to Solr. A value of 0 indicates an infinite timeout.")
@ -155,6 +178,8 @@ public abstract class SolrProcessor extends AbstractProcessor {
final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger(); final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger();
final Integer maxConnectionsPerHost = context.getProperty(SOLR_MAX_CONNECTIONS_PER_HOST).asInteger(); final Integer maxConnectionsPerHost = context.getProperty(SOLR_MAX_CONNECTIONS_PER_HOST).asInteger();
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
final String jaasClientAppName = context.getProperty(JAAS_CLIENT_APP_NAME).getValue();
final ModifiableSolrParams params = new ModifiableSolrParams(); final ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout); params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
@ -162,8 +187,21 @@ public abstract class SolrProcessor extends AbstractProcessor {
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections); params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost); params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
// has to happen before the client is created below so that correct configurer would be set if neeeded
if (!StringUtils.isEmpty(jaasClientAppName)) {
System.setProperty("solr.kerberos.jaas.appname", jaasClientAppName);
HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
}
final HttpClient httpClient = HttpClientUtil.createClient(params); final HttpClient httpClient = HttpClientUtil.createClient(params);
if (sslContextService != null) {
final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
final SSLSocketFactory sslSocketFactory = new SSLSocketFactory(sslContext);
final Scheme httpsScheme = new Scheme("https", 443, sslSocketFactory);
httpClient.getConnectionManager().getSchemeRegistry().register(httpsScheme);
}
if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) { if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
return new HttpSolrClient(solrLocation, httpClient); return new HttpSolrClient(solrLocation, httpClient);
} else { } else {
@ -204,6 +242,51 @@ public abstract class SolrProcessor extends AbstractProcessor {
} }
} }
// If a JAAS Client App Name is provided then the system property for the JAAS config file must be set,
// and that config file must contain an entry for the name provided by the processor
final String jaasAppName = context.getProperty(JAAS_CLIENT_APP_NAME).getValue();
if (!StringUtils.isEmpty(jaasAppName)) {
final String loginConf = System.getProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
if (StringUtils.isEmpty(loginConf)) {
problems.add(new ValidationResult.Builder()
.subject(JAAS_CLIENT_APP_NAME.getDisplayName())
.valid(false)
.explanation("the system property " + Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP + " must be set when providing a JAAS Client App Name")
.build());
} else {
final Configuration config = javax.security.auth.login.Configuration.getConfiguration();
if (config.getAppConfigurationEntry(jaasAppName) == null) {
problems.add(new ValidationResult.Builder()
.subject(JAAS_CLIENT_APP_NAME.getDisplayName())
.valid(false)
.explanation("'" + jaasAppName + "' does not exist in " + loginConf)
.build());
}
}
}
// For solr cloud the location will be the ZooKeeper host:port so we can't validate the SSLContext, but for standard solr
// we can validate if the url starts with https we need an SSLContextService, if it starts with http we can't have an SSLContextService
if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
final String solrLocation = context.getProperty(SOLR_LOCATION).getValue();
if (solrLocation != null) {
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
if (solrLocation.startsWith("https:") && sslContextService == null) {
problems.add(new ValidationResult.Builder()
.subject(SSL_CONTEXT_SERVICE.getDisplayName())
.valid(false)
.explanation("an SSLContextService must be provided when using https")
.build());
} else if (solrLocation.startsWith("http:") && sslContextService != null) {
problems.add(new ValidationResult.Builder()
.subject(SSL_CONTEXT_SERVICE.getDisplayName())
.valid(false)
.explanation("an SSLContextService can not be provided when using http")
.build());
}
}
}
Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context); Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
if (otherProblems != null) { if (otherProblems != null) {
problems.addAll(otherProblems); problems.addAll(otherProblems);

View File

@ -22,11 +22,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Properties; import java.util.HashMap;
import java.util.Map;
/** /**
* Helper to create EmbeddedSolrServer instances for testing. * Helper to create EmbeddedSolrServer instances for testing.
@ -34,7 +34,6 @@ import java.util.Properties;
public class EmbeddedSolrServerFactory { public class EmbeddedSolrServerFactory {
public static final String DEFAULT_SOLR_HOME = "src/test/resources/solr"; public static final String DEFAULT_SOLR_HOME = "src/test/resources/solr";
public static final String DEFAULT_CORE_HOME = "src/test/resources/";
public static final String DEFAULT_DATA_DIR = "target"; public static final String DEFAULT_DATA_DIR = "target";
/** /**
@ -44,8 +43,7 @@ public class EmbeddedSolrServerFactory {
* @return an EmbeddedSolrServer for the given core * @return an EmbeddedSolrServer for the given core
*/ */
public static SolrClient create(String coreName) throws IOException { public static SolrClient create(String coreName) throws IOException {
return create(DEFAULT_SOLR_HOME, DEFAULT_CORE_HOME, return create(DEFAULT_SOLR_HOME, coreName, DEFAULT_DATA_DIR);
coreName, DEFAULT_DATA_DIR);
} }
/** /**
@ -59,25 +57,21 @@ public class EmbeddedSolrServerFactory {
* *
* @return an EmbeddedSolrServer for the given core * @return an EmbeddedSolrServer for the given core
*/ */
public static SolrClient create(String solrHome, String coreHome, String coreName, String dataDir) public static SolrClient create(String solrHome, String coreName, String dataDir)
throws IOException { throws IOException {
Properties props = new Properties(); Map<String,String> props = new HashMap<>();
if (dataDir != null) { if (dataDir != null) {
File coreDataDir = new File(dataDir + "/" + coreName); File coreDataDir = new File(dataDir + "/" + coreName);
if (coreDataDir.exists()) { if (coreDataDir.exists()) {
FileUtils.deleteDirectory(coreDataDir); FileUtils.deleteDirectory(coreDataDir);
} }
props.setProperty("dataDir", dataDir + "/" + coreName); props.put("dataDir", dataDir + "/" + coreName);
} }
CoreContainer coreContainer = new CoreContainer(solrHome); final CoreContainer coreContainer = new CoreContainer(solrHome);
coreContainer.load(); coreContainer.load();
CoreDescriptor descriptor = new CoreDescriptor(coreContainer, coreName,
new File(coreHome, coreName).getAbsolutePath(), props);
coreContainer.create(descriptor);
return new EmbeddedSolrServer(coreContainer, coreName); return new EmbeddedSolrServer(coreContainer, coreName);
} }
} }

View File

@ -63,7 +63,7 @@ public class TestGetSolr {
.getLocation().getFile() + "../../target"; .getLocation().getFile() + "../../target";
solrClient = EmbeddedSolrServerFactory.create(EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME, solrClient = EmbeddedSolrServerFactory.create(EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME,
EmbeddedSolrServerFactory.DEFAULT_CORE_HOME, DEFAULT_SOLR_CORE, relPath); DEFAULT_SOLR_CORE, relPath);
// create some test documents // create some test documents
SolrInputDocument doc1 = new SolrInputDocument(); SolrInputDocument doc1 = new SolrInputDocument();

View File

@ -16,7 +16,11 @@
*/ */
package org.apache.nifi.processors.solr; package org.apache.nifi.processors.solr;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners; import org.apache.nifi.util.TestRunners;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrClient;
@ -24,15 +28,19 @@ import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
@ -354,6 +362,124 @@ public class TestPutSolrContentStream {
runner.assertValid(); runner.assertValid();
} }
@Test
public void testHttpsUrlShouldRequireSSLContext() throws InitializationException {
final TestRunner runner = TestRunners.newTestRunner(PutSolrContentStream.class);
runner.setProperty(PutSolrContentStream.SOLR_TYPE, PutSolrContentStream.SOLR_TYPE_STANDARD.getValue());
runner.setProperty(PutSolrContentStream.SOLR_LOCATION, "https://localhost:8443/solr");
runner.assertNotValid();
final SSLContextService sslContextService = new MockSSLContextService();
runner.addControllerService("ssl-context", sslContextService);
runner.enableControllerService(sslContextService);
runner.setProperty(PutSolrContentStream.SSL_CONTEXT_SERVICE, "ssl-context");
runner.assertValid();
}
@Test
public void testHttpUrlShouldNotAllowSSLContext() throws InitializationException {
final TestRunner runner = TestRunners.newTestRunner(PutSolrContentStream.class);
runner.setProperty(PutSolrContentStream.SOLR_TYPE, PutSolrContentStream.SOLR_TYPE_STANDARD.getValue());
runner.setProperty(PutSolrContentStream.SOLR_LOCATION, "http://localhost:8443/solr");
runner.assertValid();
final SSLContextService sslContextService = new MockSSLContextService();
runner.addControllerService("ssl-context", sslContextService);
runner.enableControllerService(sslContextService);
runner.setProperty(PutSolrContentStream.SSL_CONTEXT_SERVICE, "ssl-context");
runner.assertNotValid();
}
@Test
public void testJAASClientAppNameValidation() {
final TestRunner runner = TestRunners.newTestRunner(PutSolrContentStream.class);
runner.setProperty(PutSolrContentStream.SOLR_TYPE, PutSolrContentStream.SOLR_TYPE_STANDARD.getValue());
runner.setProperty(PutSolrContentStream.SOLR_LOCATION, "http://localhost:8443/solr");
runner.assertValid();
// clear the jaas config system property if it was set
final String jaasConfig = System.getProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
if (!StringUtils.isEmpty(jaasConfig)) {
System.clearProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
}
// should be invalid if we have a client name but not config file
runner.setProperty(PutSolrContentStream.JAAS_CLIENT_APP_NAME, "Client");
runner.assertNotValid();
// should be invalid if we have a client name that is not in the config file
final File jaasConfigFile = new File("src/test/resources/jaas-client.conf");
System.setProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP, jaasConfigFile.getAbsolutePath());
runner.assertNotValid();
// should be valid now that the name matches up with the config file
runner.setProperty(PutSolrContentStream.JAAS_CLIENT_APP_NAME, "SolrJClient");
runner.assertValid();
}
/**
* Mock implementation so we don't need to have a real keystore/truststore available for testing.
*/
private class MockSSLContextService extends AbstractControllerService implements SSLContextService {
@Override
public SSLContext createSSLContext(ClientAuth clientAuth) throws ProcessException {
return null;
}
@Override
public String getTrustStoreFile() {
return null;
}
@Override
public String getTrustStoreType() {
return null;
}
@Override
public String getTrustStorePassword() {
return null;
}
@Override
public boolean isTrustStoreConfigured() {
return false;
}
@Override
public String getKeyStoreFile() {
return null;
}
@Override
public String getKeyStoreType() {
return null;
}
@Override
public String getKeyStorePassword() {
return null;
}
@Override
public String getKeyPassword() {
return null;
}
@Override
public boolean isKeyStoreConfigured() {
return false;
}
@Override
public String getSslAlgorithm() {
return null;
}
}
// Override the createSolrClient method to inject a custom SolrClient. // Override the createSolrClient method to inject a custom SolrClient.
private class CollectionVerifyingProcessor extends PutSolrContentStream { private class CollectionVerifyingProcessor extends PutSolrContentStream {
@ -375,7 +501,7 @@ public class TestPutSolrContentStream {
} }
@Override @Override
public void shutdown() { public void close() {
} }
@ -432,7 +558,6 @@ public class TestPutSolrContentStream {
return EmbeddedSolrServerFactory.create( return EmbeddedSolrServerFactory.create(
EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME, EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME,
EmbeddedSolrServerFactory.DEFAULT_CORE_HOME,
coreName, relPath); coreName, relPath);
} }

View File

@ -0,0 +1,9 @@
SolrJClient {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
keyTab="krb5.keytab"
storeKey=true
useTicketCache=false
debug=true
principal="nifi@SOLR.ORG";
};

View File

@ -28,7 +28,7 @@
<description>A bundle of processors that can store and retrieve data from Apache Solr</description> <description>A bundle of processors that can store and retrieve data from Apache Solr</description>
<properties> <properties>
<solr.version>5.1.0</solr.version> <solr.version>6.2.0</solr.version>
</properties> </properties>
<modules> <modules>
@ -38,11 +38,11 @@
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
<!-- overriding these for solrj which needs 4.3.x --> <!-- overriding these for solrj which needs 4.4.x -->
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
<version>4.3.1</version> <version>4.4.1</version>
</dependency> </dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>