mirror of https://github.com/apache/nifi.git
NIFI-1768 Adding TLS/SSL support to Solr processors NIFI-1980 Added a default value for PutSolrContentStream commitWithIn NIFI-2568 Added Kerberos support to Solr processors
Upgrading SolrJ to 6.2. Signed-off-by: Yolanda M. Davis <ymdavis@apache.org> This closes #1005
This commit is contained in:
parent
c4826cac54
commit
d4948a3778
|
@ -35,6 +35,11 @@
|
||||||
<artifactId>nifi-solr-processors</artifactId>
|
<artifactId>nifi-solr-processors</artifactId>
|
||||||
<version>1.1.0-SNAPSHOT</version>
|
<version>1.1.0-SNAPSHOT</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-standard-services-api-nar</artifactId>
|
||||||
|
<type>nar</type>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -37,6 +37,23 @@ The following binary components are provided under the Apache Software License v
|
||||||
This product includes software from the Spring Framework,
|
This product includes software from the Spring Framework,
|
||||||
under the Apache License 2.0 (see: StringUtils.containsWhitespace())
|
under the Apache License 2.0 (see: StringUtils.containsWhitespace())
|
||||||
|
|
||||||
|
(ASLv2) Apache Commons Codec
|
||||||
|
The following NOTICE information applies:
|
||||||
|
Apache Commons Codec
|
||||||
|
Copyright 2002-2014 The Apache Software Foundation
|
||||||
|
|
||||||
|
src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
|
||||||
|
contains test data from http://aspell.net/test/orig/batch0.tab.
|
||||||
|
Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
|
||||||
|
|
||||||
|
===============================================================================
|
||||||
|
|
||||||
|
The content of package org.apache.commons.codec.language.bm has been translated
|
||||||
|
from the original php source code available at http://stevemorse.org/phoneticinfo.htm
|
||||||
|
with permission from the original authors.
|
||||||
|
Original source copyright:
|
||||||
|
Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
|
||||||
|
|
||||||
(ASLv2) Apache ZooKeeper
|
(ASLv2) Apache ZooKeeper
|
||||||
The following NOTICE information applies:
|
The following NOTICE information applies:
|
||||||
Apache ZooKeeper
|
Apache ZooKeeper
|
||||||
|
|
|
@ -31,6 +31,11 @@
|
||||||
<artifactId>solr-solrj</artifactId>
|
<artifactId>solr-solrj</artifactId>
|
||||||
<version>${solr.version}</version>
|
<version>${solr.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>commons-codec</groupId>
|
||||||
|
<artifactId>commons-codec</artifactId>
|
||||||
|
<version>1.10</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-api</artifactId>
|
<artifactId>nifi-api</artifactId>
|
||||||
|
@ -39,6 +44,11 @@
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-processor-utils</artifactId>
|
<artifactId>nifi-processor-utils</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-ssl-context-service-api</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
<!-- test dependencies -->
|
<!-- test dependencies -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
@ -68,8 +78,7 @@
|
||||||
<version>${solr.version}</version>
|
<version>${solr.version}</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
<!-- These Lucene deps should be brought in through solr-core, but there
|
<!-- Need to declare the newer versions of these b/c NiFi uses Lucene 4.10.3 -->
|
||||||
appears to be an issue with 5.0.0 that still references some 4.10.3 poms -->
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.lucene</groupId>
|
<groupId>org.apache.lucene</groupId>
|
||||||
<artifactId>lucene-core</artifactId>
|
<artifactId>lucene-core</artifactId>
|
||||||
|
@ -97,18 +106,19 @@
|
||||||
<configuration>
|
<configuration>
|
||||||
<excludes combine.children="append">
|
<excludes combine.children="append">
|
||||||
<exclude>src/test/resources/solr/solr.xml</exclude>
|
<exclude>src/test/resources/solr/solr.xml</exclude>
|
||||||
<exclude>src/test/resources/testCollection/core.properties</exclude>
|
<exclude>src/test/resources/solr/testCollection/core.properties</exclude>
|
||||||
<exclude>src/test/resources/testCollection/conf/_rest_managed.json</exclude>
|
<exclude>src/test/resources/solr/testCollection/conf/_rest_managed.json</exclude>
|
||||||
<exclude>src/test/resources/testCollection/conf/protowords.txt</exclude>
|
<exclude>src/test/resources/solr/testCollection/conf/protowords.txt</exclude>
|
||||||
<exclude>src/test/resources/testCollection/conf/schema.xml</exclude>
|
<exclude>src/test/resources/solr/testCollection/conf/schema.xml</exclude>
|
||||||
<exclude>src/test/resources/testCollection/conf/solrconfig.xml</exclude>
|
<exclude>src/test/resources/solr/testCollection/conf/solrconfig.xml</exclude>
|
||||||
<exclude>src/test/resources/testCollection/conf/synonyms.txt</exclude>
|
<exclude>src/test/resources/solr/testCollection/conf/synonyms.txt</exclude>
|
||||||
<exclude>src/test/resources/testCollection/conf/lang/stopwords_en.txt</exclude>
|
<exclude>src/test/resources/solr/testCollection/conf/lang/stopwords_en.txt</exclude>
|
||||||
<exclude>src/test/resources/testdata/test-csv-multiple-docs.csv</exclude>
|
<exclude>src/test/resources/testdata/test-csv-multiple-docs.csv</exclude>
|
||||||
<exclude>src/test/resources/testdata/test-custom-json-single-doc.json</exclude>
|
<exclude>src/test/resources/testdata/test-custom-json-single-doc.json</exclude>
|
||||||
<exclude>src/test/resources/testdata/test-solr-json-multiple-docs.json</exclude>
|
<exclude>src/test/resources/testdata/test-solr-json-multiple-docs.json</exclude>
|
||||||
<exclude>src/test/resources/testdata/test-xml-multiple-docs.xml</exclude>
|
<exclude>src/test/resources/testdata/test-xml-multiple-docs.xml</exclude>
|
||||||
<exclude>src/test/resources/log4j.properties</exclude>
|
<exclude>src/test/resources/log4j.properties</exclude>
|
||||||
|
<exclude>src/test/resources/jaas-client.conf</exclude>
|
||||||
</excludes>
|
</excludes>
|
||||||
</configuration>
|
</configuration>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
|
|
@ -23,6 +23,7 @@ import java.io.FileInputStream;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
@ -47,6 +48,7 @@ import org.apache.nifi.annotation.lifecycle.OnRemoved;
|
||||||
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
||||||
import org.apache.nifi.components.PropertyDescriptor;
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
import org.apache.nifi.flowfile.FlowFile;
|
import org.apache.nifi.flowfile.FlowFile;
|
||||||
|
import org.apache.nifi.flowfile.attributes.CoreAttributes;
|
||||||
import org.apache.nifi.logging.ComponentLog;
|
import org.apache.nifi.logging.ComponentLog;
|
||||||
import org.apache.nifi.processor.ProcessContext;
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
import org.apache.nifi.processor.ProcessSession;
|
import org.apache.nifi.processor.ProcessSession;
|
||||||
|
@ -62,6 +64,7 @@ import org.apache.solr.client.solrj.response.QueryResponse;
|
||||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||||
import org.apache.solr.common.SolrDocument;
|
import org.apache.solr.common.SolrDocument;
|
||||||
import org.apache.solr.common.SolrDocumentList;
|
import org.apache.solr.common.SolrDocumentList;
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
|
||||||
@Tags({"Apache", "Solr", "Get", "Pull"})
|
@Tags({"Apache", "Solr", "Get", "Pull"})
|
||||||
@InputRequirement(Requirement.INPUT_FORBIDDEN)
|
@InputRequirement(Requirement.INPUT_FORBIDDEN)
|
||||||
|
@ -139,6 +142,8 @@ public class GetSolr extends SolrProcessor {
|
||||||
descriptors.add(SORT_CLAUSE);
|
descriptors.add(SORT_CLAUSE);
|
||||||
descriptors.add(DATE_FIELD);
|
descriptors.add(DATE_FIELD);
|
||||||
descriptors.add(BATCH_SIZE);
|
descriptors.add(BATCH_SIZE);
|
||||||
|
descriptors.add(JAAS_CLIENT_APP_NAME);
|
||||||
|
descriptors.add(SSL_CONTEXT_SERVICE);
|
||||||
descriptors.add(SOLR_SOCKET_TIMEOUT);
|
descriptors.add(SOLR_SOCKET_TIMEOUT);
|
||||||
descriptors.add(SOLR_CONNECTION_TIMEOUT);
|
descriptors.add(SOLR_CONNECTION_TIMEOUT);
|
||||||
descriptors.add(SOLR_MAX_CONNECTIONS);
|
descriptors.add(SOLR_MAX_CONNECTIONS);
|
||||||
|
@ -235,6 +240,7 @@ public class GetSolr extends SolrProcessor {
|
||||||
if (documentList != null && documentList.getNumFound() > 0) {
|
if (documentList != null && documentList.getNumFound() > 0) {
|
||||||
FlowFile flowFile = session.create();
|
FlowFile flowFile = session.create();
|
||||||
flowFile = session.write(flowFile, new QueryResponseOutputStreamCallback(response));
|
flowFile = session.write(flowFile, new QueryResponseOutputStreamCallback(response));
|
||||||
|
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/xml");
|
||||||
session.transfer(flowFile, REL_SUCCESS);
|
session.transfer(flowFile, REL_SUCCESS);
|
||||||
|
|
||||||
StringBuilder transitUri = new StringBuilder("solr://");
|
StringBuilder transitUri = new StringBuilder("solr://");
|
||||||
|
@ -324,9 +330,19 @@ public class GetSolr extends SolrProcessor {
|
||||||
@Override
|
@Override
|
||||||
public void process(OutputStream out) throws IOException {
|
public void process(OutputStream out) throws IOException {
|
||||||
for (SolrDocument doc : response.getResults()) {
|
for (SolrDocument doc : response.getResults()) {
|
||||||
String xml = ClientUtils.toXML(ClientUtils.toSolrInputDocument(doc));
|
String xml = ClientUtils.toXML(toSolrInputDocument(doc));
|
||||||
IOUtils.write(xml, out);
|
IOUtils.write(xml, out, StandardCharsets.UTF_8);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public SolrInputDocument toSolrInputDocument(SolrDocument d) {
|
||||||
|
SolrInputDocument doc = new SolrInputDocument();
|
||||||
|
|
||||||
|
for (String name : d.getFieldNames()) {
|
||||||
|
doc.addField(name, d.getFieldValue(name));
|
||||||
|
}
|
||||||
|
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -88,6 +88,7 @@ public class PutSolrContentStream extends SolrProcessor {
|
||||||
.required(false)
|
.required(false)
|
||||||
.addValidator(StandardValidators.POSITIVE_LONG_VALIDATOR)
|
.addValidator(StandardValidators.POSITIVE_LONG_VALIDATOR)
|
||||||
.expressionLanguageSupported(true)
|
.expressionLanguageSupported(true)
|
||||||
|
.defaultValue("5000")
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
public static final Relationship REL_SUCCESS = new Relationship.Builder()
|
||||||
|
@ -123,6 +124,8 @@ public class PutSolrContentStream extends SolrProcessor {
|
||||||
descriptors.add(CONTENT_STREAM_PATH);
|
descriptors.add(CONTENT_STREAM_PATH);
|
||||||
descriptors.add(CONTENT_TYPE);
|
descriptors.add(CONTENT_TYPE);
|
||||||
descriptors.add(COMMIT_WITHIN);
|
descriptors.add(COMMIT_WITHIN);
|
||||||
|
descriptors.add(JAAS_CLIENT_APP_NAME);
|
||||||
|
descriptors.add(SSL_CONTEXT_SERVICE);
|
||||||
descriptors.add(SOLR_SOCKET_TIMEOUT);
|
descriptors.add(SOLR_SOCKET_TIMEOUT);
|
||||||
descriptors.add(SOLR_CONNECTION_TIMEOUT);
|
descriptors.add(SOLR_CONNECTION_TIMEOUT);
|
||||||
descriptors.add(SOLR_MAX_CONNECTIONS);
|
descriptors.add(SOLR_MAX_CONNECTIONS);
|
||||||
|
|
|
@ -18,7 +18,10 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.nifi.processors.solr;
|
package org.apache.nifi.processors.solr;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.http.client.HttpClient;
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.conn.scheme.Scheme;
|
||||||
|
import org.apache.http.conn.ssl.SSLSocketFactory;
|
||||||
import org.apache.nifi.annotation.lifecycle.OnScheduled;
|
import org.apache.nifi.annotation.lifecycle.OnScheduled;
|
||||||
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
import org.apache.nifi.annotation.lifecycle.OnStopped;
|
||||||
import org.apache.nifi.components.AllowableValue;
|
import org.apache.nifi.components.AllowableValue;
|
||||||
|
@ -28,12 +31,16 @@ import org.apache.nifi.components.ValidationResult;
|
||||||
import org.apache.nifi.processor.AbstractProcessor;
|
import org.apache.nifi.processor.AbstractProcessor;
|
||||||
import org.apache.nifi.processor.ProcessContext;
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
import org.apache.nifi.processor.util.StandardValidators;
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
import org.apache.nifi.ssl.SSLContextService;
|
||||||
import org.apache.solr.client.solrj.SolrClient;
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
import org.apache.solr.client.solrj.impl.HttpClientUtil;
|
import org.apache.solr.client.solrj.impl.HttpClientUtil;
|
||||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
|
||||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||||
|
|
||||||
|
import javax.net.ssl.SSLContext;
|
||||||
|
import javax.security.auth.login.Configuration;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
@ -76,6 +83,22 @@ public abstract class SolrProcessor extends AbstractProcessor {
|
||||||
.expressionLanguageSupported(true)
|
.expressionLanguageSupported(true)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor JAAS_CLIENT_APP_NAME = new PropertyDescriptor
|
||||||
|
.Builder().name("JAAS Client App Name")
|
||||||
|
.description("The name of the JAAS configuration entry to use when performing Kerberos authentication to Solr. If this property is " +
|
||||||
|
"not provided, Kerberos authentication will not be attempted. The value must match an entry in the file specified by the " +
|
||||||
|
"system property java.security.auth.login.config.")
|
||||||
|
.required(false)
|
||||||
|
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
|
||||||
|
.name("SSL Context Service")
|
||||||
|
.description("The Controller Service to use in order to obtain an SSL Context. This property must be set when communicating with a Solr over https.")
|
||||||
|
.required(false)
|
||||||
|
.identifiesControllerService(SSLContextService.class)
|
||||||
|
.build();
|
||||||
|
|
||||||
public static final PropertyDescriptor SOLR_SOCKET_TIMEOUT = new PropertyDescriptor
|
public static final PropertyDescriptor SOLR_SOCKET_TIMEOUT = new PropertyDescriptor
|
||||||
.Builder().name("Solr Socket Timeout")
|
.Builder().name("Solr Socket Timeout")
|
||||||
.description("The amount of time to wait for data on a socket connection to Solr. A value of 0 indicates an infinite timeout.")
|
.description("The amount of time to wait for data on a socket connection to Solr. A value of 0 indicates an infinite timeout.")
|
||||||
|
@ -155,6 +178,8 @@ public abstract class SolrProcessor extends AbstractProcessor {
|
||||||
final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
|
final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
|
||||||
final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger();
|
final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger();
|
||||||
final Integer maxConnectionsPerHost = context.getProperty(SOLR_MAX_CONNECTIONS_PER_HOST).asInteger();
|
final Integer maxConnectionsPerHost = context.getProperty(SOLR_MAX_CONNECTIONS_PER_HOST).asInteger();
|
||||||
|
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
|
||||||
|
final String jaasClientAppName = context.getProperty(JAAS_CLIENT_APP_NAME).getValue();
|
||||||
|
|
||||||
final ModifiableSolrParams params = new ModifiableSolrParams();
|
final ModifiableSolrParams params = new ModifiableSolrParams();
|
||||||
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
|
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
|
||||||
|
@ -162,8 +187,21 @@ public abstract class SolrProcessor extends AbstractProcessor {
|
||||||
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
|
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
|
||||||
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
|
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
|
||||||
|
|
||||||
|
// has to happen before the client is created below so that correct configurer would be set if neeeded
|
||||||
|
if (!StringUtils.isEmpty(jaasClientAppName)) {
|
||||||
|
System.setProperty("solr.kerberos.jaas.appname", jaasClientAppName);
|
||||||
|
HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
|
||||||
|
}
|
||||||
|
|
||||||
final HttpClient httpClient = HttpClientUtil.createClient(params);
|
final HttpClient httpClient = HttpClientUtil.createClient(params);
|
||||||
|
|
||||||
|
if (sslContextService != null) {
|
||||||
|
final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
|
||||||
|
final SSLSocketFactory sslSocketFactory = new SSLSocketFactory(sslContext);
|
||||||
|
final Scheme httpsScheme = new Scheme("https", 443, sslSocketFactory);
|
||||||
|
httpClient.getConnectionManager().getSchemeRegistry().register(httpsScheme);
|
||||||
|
}
|
||||||
|
|
||||||
if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
|
if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
|
||||||
return new HttpSolrClient(solrLocation, httpClient);
|
return new HttpSolrClient(solrLocation, httpClient);
|
||||||
} else {
|
} else {
|
||||||
|
@ -204,6 +242,51 @@ public abstract class SolrProcessor extends AbstractProcessor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If a JAAS Client App Name is provided then the system property for the JAAS config file must be set,
|
||||||
|
// and that config file must contain an entry for the name provided by the processor
|
||||||
|
final String jaasAppName = context.getProperty(JAAS_CLIENT_APP_NAME).getValue();
|
||||||
|
if (!StringUtils.isEmpty(jaasAppName)) {
|
||||||
|
final String loginConf = System.getProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
|
||||||
|
if (StringUtils.isEmpty(loginConf)) {
|
||||||
|
problems.add(new ValidationResult.Builder()
|
||||||
|
.subject(JAAS_CLIENT_APP_NAME.getDisplayName())
|
||||||
|
.valid(false)
|
||||||
|
.explanation("the system property " + Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP + " must be set when providing a JAAS Client App Name")
|
||||||
|
.build());
|
||||||
|
} else {
|
||||||
|
final Configuration config = javax.security.auth.login.Configuration.getConfiguration();
|
||||||
|
if (config.getAppConfigurationEntry(jaasAppName) == null) {
|
||||||
|
problems.add(new ValidationResult.Builder()
|
||||||
|
.subject(JAAS_CLIENT_APP_NAME.getDisplayName())
|
||||||
|
.valid(false)
|
||||||
|
.explanation("'" + jaasAppName + "' does not exist in " + loginConf)
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For solr cloud the location will be the ZooKeeper host:port so we can't validate the SSLContext, but for standard solr
|
||||||
|
// we can validate if the url starts with https we need an SSLContextService, if it starts with http we can't have an SSLContextService
|
||||||
|
if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
|
||||||
|
final String solrLocation = context.getProperty(SOLR_LOCATION).getValue();
|
||||||
|
if (solrLocation != null) {
|
||||||
|
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
|
||||||
|
if (solrLocation.startsWith("https:") && sslContextService == null) {
|
||||||
|
problems.add(new ValidationResult.Builder()
|
||||||
|
.subject(SSL_CONTEXT_SERVICE.getDisplayName())
|
||||||
|
.valid(false)
|
||||||
|
.explanation("an SSLContextService must be provided when using https")
|
||||||
|
.build());
|
||||||
|
} else if (solrLocation.startsWith("http:") && sslContextService != null) {
|
||||||
|
problems.add(new ValidationResult.Builder()
|
||||||
|
.subject(SSL_CONTEXT_SERVICE.getDisplayName())
|
||||||
|
.valid(false)
|
||||||
|
.explanation("an SSLContextService can not be provided when using http")
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
|
Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
|
||||||
if (otherProblems != null) {
|
if (otherProblems != null) {
|
||||||
problems.addAll(otherProblems);
|
problems.addAll(otherProblems);
|
||||||
|
|
|
@ -22,11 +22,11 @@ import org.apache.commons.io.FileUtils;
|
||||||
import org.apache.solr.client.solrj.SolrClient;
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||||
import org.apache.solr.core.CoreContainer;
|
import org.apache.solr.core.CoreContainer;
|
||||||
import org.apache.solr.core.CoreDescriptor;
|
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Properties;
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper to create EmbeddedSolrServer instances for testing.
|
* Helper to create EmbeddedSolrServer instances for testing.
|
||||||
|
@ -34,7 +34,6 @@ import java.util.Properties;
|
||||||
public class EmbeddedSolrServerFactory {
|
public class EmbeddedSolrServerFactory {
|
||||||
|
|
||||||
public static final String DEFAULT_SOLR_HOME = "src/test/resources/solr";
|
public static final String DEFAULT_SOLR_HOME = "src/test/resources/solr";
|
||||||
public static final String DEFAULT_CORE_HOME = "src/test/resources/";
|
|
||||||
public static final String DEFAULT_DATA_DIR = "target";
|
public static final String DEFAULT_DATA_DIR = "target";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -44,8 +43,7 @@ public class EmbeddedSolrServerFactory {
|
||||||
* @return an EmbeddedSolrServer for the given core
|
* @return an EmbeddedSolrServer for the given core
|
||||||
*/
|
*/
|
||||||
public static SolrClient create(String coreName) throws IOException {
|
public static SolrClient create(String coreName) throws IOException {
|
||||||
return create(DEFAULT_SOLR_HOME, DEFAULT_CORE_HOME,
|
return create(DEFAULT_SOLR_HOME, coreName, DEFAULT_DATA_DIR);
|
||||||
coreName, DEFAULT_DATA_DIR);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -59,25 +57,21 @@ public class EmbeddedSolrServerFactory {
|
||||||
*
|
*
|
||||||
* @return an EmbeddedSolrServer for the given core
|
* @return an EmbeddedSolrServer for the given core
|
||||||
*/
|
*/
|
||||||
public static SolrClient create(String solrHome, String coreHome, String coreName, String dataDir)
|
public static SolrClient create(String solrHome, String coreName, String dataDir)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
Properties props = new Properties();
|
Map<String,String> props = new HashMap<>();
|
||||||
if (dataDir != null) {
|
if (dataDir != null) {
|
||||||
File coreDataDir = new File(dataDir + "/" + coreName);
|
File coreDataDir = new File(dataDir + "/" + coreName);
|
||||||
if (coreDataDir.exists()) {
|
if (coreDataDir.exists()) {
|
||||||
FileUtils.deleteDirectory(coreDataDir);
|
FileUtils.deleteDirectory(coreDataDir);
|
||||||
}
|
}
|
||||||
props.setProperty("dataDir", dataDir + "/" + coreName);
|
props.put("dataDir", dataDir + "/" + coreName);
|
||||||
}
|
}
|
||||||
|
|
||||||
CoreContainer coreContainer = new CoreContainer(solrHome);
|
final CoreContainer coreContainer = new CoreContainer(solrHome);
|
||||||
coreContainer.load();
|
coreContainer.load();
|
||||||
|
|
||||||
CoreDescriptor descriptor = new CoreDescriptor(coreContainer, coreName,
|
|
||||||
new File(coreHome, coreName).getAbsolutePath(), props);
|
|
||||||
|
|
||||||
coreContainer.create(descriptor);
|
|
||||||
return new EmbeddedSolrServer(coreContainer, coreName);
|
return new EmbeddedSolrServer(coreContainer, coreName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,7 +63,7 @@ public class TestGetSolr {
|
||||||
.getLocation().getFile() + "../../target";
|
.getLocation().getFile() + "../../target";
|
||||||
|
|
||||||
solrClient = EmbeddedSolrServerFactory.create(EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME,
|
solrClient = EmbeddedSolrServerFactory.create(EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME,
|
||||||
EmbeddedSolrServerFactory.DEFAULT_CORE_HOME, DEFAULT_SOLR_CORE, relPath);
|
DEFAULT_SOLR_CORE, relPath);
|
||||||
|
|
||||||
// create some test documents
|
// create some test documents
|
||||||
SolrInputDocument doc1 = new SolrInputDocument();
|
SolrInputDocument doc1 = new SolrInputDocument();
|
||||||
|
|
|
@ -16,7 +16,11 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.nifi.processors.solr;
|
package org.apache.nifi.processors.solr;
|
||||||
|
|
||||||
|
import org.apache.nifi.controller.AbstractControllerService;
|
||||||
import org.apache.nifi.processor.ProcessContext;
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
|
import org.apache.nifi.processor.exception.ProcessException;
|
||||||
|
import org.apache.nifi.reporting.InitializationException;
|
||||||
|
import org.apache.nifi.ssl.SSLContextService;
|
||||||
import org.apache.nifi.util.TestRunner;
|
import org.apache.nifi.util.TestRunner;
|
||||||
import org.apache.nifi.util.TestRunners;
|
import org.apache.nifi.util.TestRunners;
|
||||||
import org.apache.solr.client.solrj.SolrClient;
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
|
@ -24,15 +28,19 @@ import org.apache.solr.client.solrj.SolrQuery;
|
||||||
import org.apache.solr.client.solrj.SolrRequest;
|
import org.apache.solr.client.solrj.SolrRequest;
|
||||||
import org.apache.solr.client.solrj.SolrServerException;
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
|
||||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||||
import org.apache.solr.common.SolrDocument;
|
import org.apache.solr.common.SolrDocument;
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
import org.apache.solr.common.SolrInputDocument;
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
import org.apache.solr.common.StringUtils;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
import javax.net.ssl.SSLContext;
|
||||||
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -354,6 +362,124 @@ public class TestPutSolrContentStream {
|
||||||
runner.assertValid();
|
runner.assertValid();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testHttpsUrlShouldRequireSSLContext() throws InitializationException {
|
||||||
|
final TestRunner runner = TestRunners.newTestRunner(PutSolrContentStream.class);
|
||||||
|
runner.setProperty(PutSolrContentStream.SOLR_TYPE, PutSolrContentStream.SOLR_TYPE_STANDARD.getValue());
|
||||||
|
runner.setProperty(PutSolrContentStream.SOLR_LOCATION, "https://localhost:8443/solr");
|
||||||
|
runner.assertNotValid();
|
||||||
|
|
||||||
|
final SSLContextService sslContextService = new MockSSLContextService();
|
||||||
|
runner.addControllerService("ssl-context", sslContextService);
|
||||||
|
runner.enableControllerService(sslContextService);
|
||||||
|
|
||||||
|
runner.setProperty(PutSolrContentStream.SSL_CONTEXT_SERVICE, "ssl-context");
|
||||||
|
runner.assertValid();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testHttpUrlShouldNotAllowSSLContext() throws InitializationException {
|
||||||
|
final TestRunner runner = TestRunners.newTestRunner(PutSolrContentStream.class);
|
||||||
|
runner.setProperty(PutSolrContentStream.SOLR_TYPE, PutSolrContentStream.SOLR_TYPE_STANDARD.getValue());
|
||||||
|
runner.setProperty(PutSolrContentStream.SOLR_LOCATION, "http://localhost:8443/solr");
|
||||||
|
runner.assertValid();
|
||||||
|
|
||||||
|
final SSLContextService sslContextService = new MockSSLContextService();
|
||||||
|
runner.addControllerService("ssl-context", sslContextService);
|
||||||
|
runner.enableControllerService(sslContextService);
|
||||||
|
|
||||||
|
runner.setProperty(PutSolrContentStream.SSL_CONTEXT_SERVICE, "ssl-context");
|
||||||
|
runner.assertNotValid();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testJAASClientAppNameValidation() {
|
||||||
|
final TestRunner runner = TestRunners.newTestRunner(PutSolrContentStream.class);
|
||||||
|
runner.setProperty(PutSolrContentStream.SOLR_TYPE, PutSolrContentStream.SOLR_TYPE_STANDARD.getValue());
|
||||||
|
runner.setProperty(PutSolrContentStream.SOLR_LOCATION, "http://localhost:8443/solr");
|
||||||
|
runner.assertValid();
|
||||||
|
|
||||||
|
// clear the jaas config system property if it was set
|
||||||
|
final String jaasConfig = System.getProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
|
||||||
|
if (!StringUtils.isEmpty(jaasConfig)) {
|
||||||
|
System.clearProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
|
||||||
|
}
|
||||||
|
|
||||||
|
// should be invalid if we have a client name but not config file
|
||||||
|
runner.setProperty(PutSolrContentStream.JAAS_CLIENT_APP_NAME, "Client");
|
||||||
|
runner.assertNotValid();
|
||||||
|
|
||||||
|
// should be invalid if we have a client name that is not in the config file
|
||||||
|
final File jaasConfigFile = new File("src/test/resources/jaas-client.conf");
|
||||||
|
System.setProperty(Krb5HttpClientConfigurer.LOGIN_CONFIG_PROP, jaasConfigFile.getAbsolutePath());
|
||||||
|
runner.assertNotValid();
|
||||||
|
|
||||||
|
// should be valid now that the name matches up with the config file
|
||||||
|
runner.setProperty(PutSolrContentStream.JAAS_CLIENT_APP_NAME, "SolrJClient");
|
||||||
|
runner.assertValid();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mock implementation so we don't need to have a real keystore/truststore available for testing.
|
||||||
|
*/
|
||||||
|
private class MockSSLContextService extends AbstractControllerService implements SSLContextService {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SSLContext createSSLContext(ClientAuth clientAuth) throws ProcessException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getTrustStoreFile() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getTrustStoreType() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getTrustStorePassword() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isTrustStoreConfigured() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getKeyStoreFile() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getKeyStoreType() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getKeyStorePassword() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getKeyPassword() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isKeyStoreConfigured() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getSslAlgorithm() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Override the createSolrClient method to inject a custom SolrClient.
|
// Override the createSolrClient method to inject a custom SolrClient.
|
||||||
private class CollectionVerifyingProcessor extends PutSolrContentStream {
|
private class CollectionVerifyingProcessor extends PutSolrContentStream {
|
||||||
|
|
||||||
|
@ -375,7 +501,7 @@ public class TestPutSolrContentStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void shutdown() {
|
public void close() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -432,7 +558,6 @@ public class TestPutSolrContentStream {
|
||||||
|
|
||||||
return EmbeddedSolrServerFactory.create(
|
return EmbeddedSolrServerFactory.create(
|
||||||
EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME,
|
EmbeddedSolrServerFactory.DEFAULT_SOLR_HOME,
|
||||||
EmbeddedSolrServerFactory.DEFAULT_CORE_HOME,
|
|
||||||
coreName, relPath);
|
coreName, relPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
SolrJClient {
|
||||||
|
com.sun.security.auth.module.Krb5LoginModule required
|
||||||
|
useKeyTab=true
|
||||||
|
keyTab="krb5.keytab"
|
||||||
|
storeKey=true
|
||||||
|
useTicketCache=false
|
||||||
|
debug=true
|
||||||
|
principal="nifi@SOLR.ORG";
|
||||||
|
};
|
|
@ -0,0 +1 @@
|
||||||
|
name=testCollection
|
|
@ -1 +0,0 @@
|
||||||
name=jsonCollection
|
|
|
@ -28,7 +28,7 @@
|
||||||
<description>A bundle of processors that can store and retrieve data from Apache Solr</description>
|
<description>A bundle of processors that can store and retrieve data from Apache Solr</description>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<solr.version>5.1.0</solr.version>
|
<solr.version>6.2.0</solr.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<modules>
|
<modules>
|
||||||
|
@ -38,11 +38,11 @@
|
||||||
|
|
||||||
<dependencyManagement>
|
<dependencyManagement>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<!-- overriding these for solrj which needs 4.3.x -->
|
<!-- overriding these for solrj which needs 4.4.x -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.httpcomponents</groupId>
|
<groupId>org.apache.httpcomponents</groupId>
|
||||||
<artifactId>httpclient</artifactId>
|
<artifactId>httpclient</artifactId>
|
||||||
<version>4.3.1</version>
|
<version>4.4.1</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</dependencyManagement>
|
</dependencyManagement>
|
||||||
|
|
Loading…
Reference in New Issue