Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-7269 - Upgrade solrj version to 7 in nifi-solr-processors 

Remove unused imports

Use the latest solrj version(8.4.1)

Setup default schemaFactory for tests
The default schemaFactory ManagedIndexSchemaFactory creates
additional files in test's resources directory. Change it to
ClassicIndexSchemaFactory for classic behavior.

This closes #4152.

Signed-off-by: Bryan Bende <bbende@apache.org>
This commit is contained in:
Mubashir Kazia 2020-03-18 21:15:11 -04:00 committed by Bryan Bende
parent 6412097eb2
commit 9b4292024b
No known key found for this signature in database
GPG Key ID: A0DDA9ED50711C39
9 changed files with 150 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
NOTICE
View File

@ -66,9 +66,9 @@ This includes derived works from Apache Calcite available under Apache Software
and nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileTableScan
This includes derived works from Apache Solr available under Apache Software License V2. Portions of the code found in
https://github.com/apache/lucene-solr/blob/branch_6x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientConfigurer.java
Copyright 2006-2018 The Apache Software Foundation
The code can be found nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/kerberos/KerberosHttpClientConfigurer.java
https://github.com/apache/lucene-solr/blob/branch_7x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
Copyright 2006-2020 The Apache Software Foundation
The code can be found nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/kerberos/KerberosHttpClientBuilder.java
This includes derived works from Apache Hadoop available under Apache Software License V2. Portions of the code found in
https://github.com/apache/hadoop/blob/release-2.7.3-RC2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

2
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/pom.xml
View File

@ -23,7 +23,7 @@
<artifactId>nifi-solr-processors</artifactId>
<packaging>jar</packaging>
<properties>
<solr.version>6.6.6</solr.version>
<solr.version>8.4.1</solr.version>
</properties>
<dependencies>
<dependency>

119
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientBuilder.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.solr;
import java.lang.invoke.MethodHandles;
import java.security.Principal;
import java.util.Optional;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.solr.client.solrj.impl.HttpClientBuilderFactory;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class is a modified version of Krb5HttpClientBuilder that is part of SolrJ.
*
* In our case we don't want to warn about the useSubjectCreds property since we know we are going to do a
* login and will have subject creds, and we also don't want to mess the static JAAS configuration of the JVM.
*/
public class KerberosHttpClientBuilder implements HttpClientBuilderFactory {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public KerberosHttpClientBuilder() {
}
public SolrHttpClientBuilder getBuilder() {
return getBuilder(HttpClientUtil.getHttpClientBuilder());
}
public void close() {
HttpClientUtil.removeRequestInterceptor(bufferedEntityInterceptor);
}
@Override
public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> builder) {
return builder.isPresent() ? getBuilder(builder.get()) : getBuilder();
}
public SolrHttpClientBuilder getBuilder(SolrHttpClientBuilder builder) {
//Enable only SPNEGO authentication scheme.
builder.setAuthSchemeRegistryProvider(() -> {
Lookup<AuthSchemeProvider> authProviders = RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, false))
.build();
return authProviders;
});
// Get the credentials from the JAAS configuration rather than here
Credentials useJaasCreds = new Credentials() {
public String getPassword() {
return null;
}
public Principal getUserPrincipal() {
return null;
}
};
HttpClientUtil.setCookiePolicy(SolrPortAwareCookieSpecFactory.POLICY_NAME);
builder.setCookieSpecRegistryProvider(() -> {
SolrPortAwareCookieSpecFactory cookieFactory = new SolrPortAwareCookieSpecFactory();
Lookup<CookieSpecProvider> cookieRegistry = RegistryBuilder.<CookieSpecProvider> create()
.register(SolrPortAwareCookieSpecFactory.POLICY_NAME, cookieFactory).build();
return cookieRegistry;
});
builder.setDefaultCredentialsProvider(() -> {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, useJaasCreds);
return credentialsProvider;
});
HttpClientUtil.addRequestInterceptor(bufferedEntityInterceptor);
return builder;
}
// Set a buffered entity based request interceptor
private HttpRequestInterceptor bufferedEntityInterceptor = (request, context) -> {
if(request instanceof HttpEntityEnclosingRequest) {
HttpEntityEnclosingRequest enclosingRequest = ((HttpEntityEnclosingRequest) request);
HttpEntity requestEntity = enclosingRequest.getEntity();
enclosingRequest.setEntity(new BufferedHttpEntity(requestEntity));
}
};
}

92
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientConfigurer.java
View File

@ -1,92 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.solr;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthSchemeRegistry;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.impl.HttpClientConfigurer;
import org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecFactory;
import org.apache.solr.common.params.SolrParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.security.Principal;
/**
* This class is a modified version of Krb5HttpClientConfigurer that is part of SolrJ.
*
* In our case we don't want to warn about the useSubjectCreds property since we know we are going to do a
* login and will have subject creds, and we also don't want to mess the static JAAS configuration of the JVM.
*/
public class KerberosHttpClientConfigurer extends HttpClientConfigurer {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public void configure(DefaultHttpClient httpClient, SolrParams config) {
super.configure(httpClient, config);
logger.info("Setting up SPNego auth...");
//Enable only SPNEGO authentication scheme.
final AuthSchemeRegistry registry = new AuthSchemeRegistry();
registry.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, false));
httpClient.setAuthSchemes(registry);
// Get the credentials from the JAAS configuration rather than here
final Credentials useJaasCreds = new Credentials() {
public String getPassword() {
return null;
}
public Principal getUserPrincipal() {
return null;
}
};
final SolrPortAwareCookieSpecFactory cookieFactory = new SolrPortAwareCookieSpecFactory();
httpClient.getCookieSpecs().register(cookieFactory.POLICY_NAME, cookieFactory);
httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY, cookieFactory.POLICY_NAME);
httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, useJaasCreds);
httpClient.addRequestInterceptor(bufferedEntityInterceptor);
}
// Set a buffered entity based request interceptor
private HttpRequestInterceptor bufferedEntityInterceptor = new HttpRequestInterceptor() {
@Override
public void process(HttpRequest request, HttpContext context) throws HttpException,
IOException {
if(request instanceof HttpEntityEnclosingRequest) {
HttpEntityEnclosingRequest enclosingRequest = ((HttpEntityEnclosingRequest) request);
HttpEntity requestEntity = enclosingRequest.getEntity();
enclosingRequest.setEntity(new BufferedHttpEntity(requestEntity));
}
}
};
}

16
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java
View File

@ -69,10 +69,12 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
@ -246,7 +248,7 @@ public class SolrUtils {
// has to happen before the client is created below so that correct configurer would be set if needed
if (kerberosCredentialsService != null || (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword))) {
HttpClientUtil.setConfigurer(new KerberosHttpClientConfigurer());
HttpClientUtil.setHttpClientBuilder(new KerberosHttpClientBuilder().getHttpClientBuilder(Optional.empty()));
}
final HttpClient httpClient = HttpClientUtil.createClient(params);
@ -259,13 +261,21 @@ public class SolrUtils {
}
if (SOLR_TYPE_STANDARD.getValue().equals(context.getProperty(SOLR_TYPE).getValue())) {
return new HttpSolrClient(solrLocation, httpClient);
return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build();
} else {
// CloudSolrClient.Builder now requires a List of ZK addresses and znode for solr as separate parameters
final String zk[] = solrLocation.split("/");
final List zkList = Arrays.asList(zk[0].split(","));
String zkRoot = "/";
if (zk.length > 1 && ! zk[1].isEmpty()) {
zkRoot += zk[1];
}
final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue();
final Integer zkClientTimeout = context.getProperty(ZK_CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer zkConnectionTimeout = context.getProperty(ZK_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
CloudSolrClient cloudSolrClient = new CloudSolrClient(solrLocation, httpClient);
CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(zkList, Optional.of(zkRoot)).withHttpClient(httpClient).build();
cloudSolrClient.setDefaultCollection(collection);
cloudSolrClient.setZkClientTimeout(zkClientTimeout);
cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout);

4
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java
View File

@ -30,6 +30,7 @@ import org.apache.nifi.util.TestRunners;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.SolrInputDocument;
import org.junit.AfterClass;
@ -82,7 +83,8 @@ public class QuerySolrIT {
public static void setup() throws IOException, SolrServerException {
CloudSolrClient solrClient = createSolrClient();
Path currentDir = Paths.get(ZK_CONFIG_PATH);
solrClient.uploadConfig(currentDir, ZK_CONFIG_NAME);
ZkClientClusterStateProvider stateProvider = new ZkClientClusterStateProvider(SOLR_LOCATION);
stateProvider.uploadConfig(currentDir, ZK_CONFIG_NAME);
solrClient.setDefaultCollection(SOLR_COLLECTION);
if (!solrClient.getZkStateReader().getClusterState().hasCollection(SOLR_COLLECTION)) {

2
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java
View File

@ -355,7 +355,7 @@ public class TestGetSolr {
runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 1);
runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc1</field></doc></docs>";
String expectedXml = "<docs><doc><field name=\"id\">doc1</field></doc></docs>";
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(GetSolr.REL_SUCCESS).get(0)))));
}

14
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java
View File

@ -406,7 +406,7 @@ public class TestQuerySolr {
runner.run();
runner.assertTransferCount(QuerySolr.RESULTS, 1);
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc2</field></doc><doc boost=\"1.0\"><field name=\"id\">doc1</field></doc></docs>";
String expectedXml = "<docs><doc><field name=\"id\">doc2</field></doc><doc><field name=\"id\">doc1</field></doc></docs>";
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
solrClient.close();
@ -425,7 +425,7 @@ public class TestQuerySolr {
runner.run();
runner.assertTransferCount(QuerySolr.RESULTS, 1);
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc2</field></doc><doc boost=\"1.0\"><field name=\"id\">doc3</field></doc></docs>";
String expectedXml = "<docs><doc><field name=\"id\">doc2</field></doc><doc><field name=\"id\">doc3</field></doc></docs>";
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
solrClient.close();
@ -447,7 +447,7 @@ public class TestQuerySolr {
runner.run();
runner.assertTransferCount(QuerySolr.RESULTS, 1);
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc2</field></doc><doc boost=\"1.0\"><field name=\"id\">doc3</field></doc></docs>";
String expectedXml = "<docs><doc><field name=\"id\">doc2</field></doc><doc><field name=\"id\">doc3</field></doc></docs>";
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
solrClient.close();
@ -471,7 +471,7 @@ public class TestQuerySolr {
flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc1</field></doc><doc boost=\"1.0\"><field name=\"id\">doc0</field></doc></docs>";
String expectedXml = "<docs><doc><field name=\"id\">doc1</field></doc><doc><field name=\"id\">doc0</field></doc></docs>";
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(flowFile))));
solrClient.close();
@ -492,7 +492,7 @@ public class TestQuerySolr {
runner.assertTransferCount(QuerySolr.RESULTS, 1);
runner.assertTransferCount(QuerySolr.ORIGINAL, 1);
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc0</field></doc></docs>";
String expectedXml = "<docs><doc><field name=\"id\">doc0</field></doc></docs>";
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
assertEquals(content, new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.ORIGINAL).get(0))));
@ -526,9 +526,9 @@ public class TestQuerySolr {
startParam += 2;
StringBuffer expectedXml = new StringBuffer()
.append("<docs><doc boost=\"1.0\"><field name=\"id\">doc")
.append("<docs><doc><field name=\"id\">doc")
.append(documentCounter++)
.append("</field></doc><doc boost=\"1.0\"><field name=\"id\">doc")
.append("</field></doc><doc><field name=\"id\">doc")
.append(documentCounter++)
.append("</field></doc></docs>");
assertThat(expectedXml.toString(), CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(flowFile))));

4
nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/resources/solr/testCollection/conf/solrconfig.xml
View File

@ -17,4 +17,6 @@
<requestHandler name="/select" class="solr.SearchHandler" />
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
</config>
<schemaFactory class="ClassicIndexSchemaFactory" />
</config>