diff --git a/NOTICE b/NOTICE index 70b9295b50..f8e747c262 100644 --- a/NOTICE +++ b/NOTICE @@ -66,9 +66,9 @@ This includes derived works from Apache Calcite available under Apache Software and nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileTableScan This includes derived works from Apache Solr available under Apache Software License V2. Portions of the code found in - https://github.com/apache/lucene-solr/blob/branch_6x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientConfigurer.java - Copyright 2006-2018 The Apache Software Foundation - The code can be found nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/kerberos/KerberosHttpClientConfigurer.java + https://github.com/apache/lucene-solr/blob/branch_7x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java + Copyright 2006-2020 The Apache Software Foundation + The code can be found nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/kerberos/KerberosHttpClientBuilder.java This includes derived works from Apache Hadoop available under Apache Software License V2. Portions of the code found in https://github.com/apache/hadoop/blob/release-2.7.3-RC2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/pom.xml b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/pom.xml index 2fb76f82d5..8bedf5dad8 100755 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/pom.xml +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/pom.xml @@ -23,7 +23,7 @@ nifi-solr-processors jar - 6.6.6 + 8.4.1 diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientBuilder.java b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientBuilder.java new file mode 100644 index 0000000000..8f2f10c0f5 --- /dev/null +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientBuilder.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.solr; + +import java.lang.invoke.MethodHandles; +import java.security.Principal; +import java.util.Optional; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.config.Lookup; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.cookie.CookieSpecProvider; +import org.apache.http.entity.BufferedHttpEntity; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.solr.client.solrj.impl.HttpClientBuilderFactory; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; +import org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is a modified version of Krb5HttpClientBuilder that is part of SolrJ. + * + * In our case we don't want to warn about the useSubjectCreds property since we know we are going to do a + * login and will have subject creds, and we also don't want to mess the static JAAS configuration of the JVM. + */ +public class KerberosHttpClientBuilder implements HttpClientBuilderFactory { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public KerberosHttpClientBuilder() { + + } + + public SolrHttpClientBuilder getBuilder() { + return getBuilder(HttpClientUtil.getHttpClientBuilder()); + } + + public void close() { + HttpClientUtil.removeRequestInterceptor(bufferedEntityInterceptor); + } + + @Override + public SolrHttpClientBuilder getHttpClientBuilder(Optional builder) { + return builder.isPresent() ? getBuilder(builder.get()) : getBuilder(); + } + + public SolrHttpClientBuilder getBuilder(SolrHttpClientBuilder builder) { + + //Enable only SPNEGO authentication scheme. + + builder.setAuthSchemeRegistryProvider(() -> { + Lookup authProviders = RegistryBuilder.create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, false)) + .build(); + return authProviders; + }); + // Get the credentials from the JAAS configuration rather than here + Credentials useJaasCreds = new Credentials() { + public String getPassword() { + return null; + } + public Principal getUserPrincipal() { + return null; + } + }; + + HttpClientUtil.setCookiePolicy(SolrPortAwareCookieSpecFactory.POLICY_NAME); + + builder.setCookieSpecRegistryProvider(() -> { + SolrPortAwareCookieSpecFactory cookieFactory = new SolrPortAwareCookieSpecFactory(); + + Lookup cookieRegistry = RegistryBuilder. create() + .register(SolrPortAwareCookieSpecFactory.POLICY_NAME, cookieFactory).build(); + + return cookieRegistry; + }); + + builder.setDefaultCredentialsProvider(() -> { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, useJaasCreds); + return credentialsProvider; + }); + HttpClientUtil.addRequestInterceptor(bufferedEntityInterceptor); + return builder; + } + + // Set a buffered entity based request interceptor + private HttpRequestInterceptor bufferedEntityInterceptor = (request, context) -> { + if(request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest enclosingRequest = ((HttpEntityEnclosingRequest) request); + HttpEntity requestEntity = enclosingRequest.getEntity(); + enclosingRequest.setEntity(new BufferedHttpEntity(requestEntity)); + } + }; +} diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientConfigurer.java b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientConfigurer.java deleted file mode 100644 index 2aa7ed16b5..0000000000 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientConfigurer.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.solr; - -import org.apache.http.HttpEntity; -import org.apache.http.HttpEntityEnclosingRequest; -import org.apache.http.HttpException; -import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; -import org.apache.http.auth.AuthSchemeRegistry; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.Credentials; -import org.apache.http.client.config.AuthSchemes; -import org.apache.http.client.params.ClientPNames; -import org.apache.http.entity.BufferedHttpEntity; -import org.apache.http.impl.auth.SPNegoSchemeFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.protocol.HttpContext; -import org.apache.solr.client.solrj.impl.HttpClientConfigurer; -import org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecFactory; -import org.apache.solr.common.params.SolrParams; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.security.Principal; - -/** - * This class is a modified version of Krb5HttpClientConfigurer that is part of SolrJ. - * - * In our case we don't want to warn about the useSubjectCreds property since we know we are going to do a - * login and will have subject creds, and we also don't want to mess the static JAAS configuration of the JVM. - */ -public class KerberosHttpClientConfigurer extends HttpClientConfigurer { - - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public void configure(DefaultHttpClient httpClient, SolrParams config) { - super.configure(httpClient, config); - logger.info("Setting up SPNego auth..."); - - //Enable only SPNEGO authentication scheme. - final AuthSchemeRegistry registry = new AuthSchemeRegistry(); - registry.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, false)); - httpClient.setAuthSchemes(registry); - - // Get the credentials from the JAAS configuration rather than here - final Credentials useJaasCreds = new Credentials() { - public String getPassword() { - return null; - } - public Principal getUserPrincipal() { - return null; - } - }; - - final SolrPortAwareCookieSpecFactory cookieFactory = new SolrPortAwareCookieSpecFactory(); - httpClient.getCookieSpecs().register(cookieFactory.POLICY_NAME, cookieFactory); - httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY, cookieFactory.POLICY_NAME); - httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, useJaasCreds); - httpClient.addRequestInterceptor(bufferedEntityInterceptor); - } - - // Set a buffered entity based request interceptor - private HttpRequestInterceptor bufferedEntityInterceptor = new HttpRequestInterceptor() { - @Override - public void process(HttpRequest request, HttpContext context) throws HttpException, - IOException { - if(request instanceof HttpEntityEnclosingRequest) { - HttpEntityEnclosingRequest enclosingRequest = ((HttpEntityEnclosingRequest) request); - HttpEntity requestEntity = enclosingRequest.getEntity(); - enclosingRequest.setEntity(new BufferedHttpEntity(requestEntity)); - } - } - }; - -} diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java index 35d44e9a84..3eabfe93e0 100644 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java @@ -69,10 +69,12 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.TimeUnit; @@ -246,7 +248,7 @@ public class SolrUtils { // has to happen before the client is created below so that correct configurer would be set if needed if (kerberosCredentialsService != null || (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword))) { - HttpClientUtil.setConfigurer(new KerberosHttpClientConfigurer()); + HttpClientUtil.setHttpClientBuilder(new KerberosHttpClientBuilder().getHttpClientBuilder(Optional.empty())); } final HttpClient httpClient = HttpClientUtil.createClient(params); @@ -259,13 +261,21 @@ public class SolrUtils { } if (SOLR_TYPE_STANDARD.getValue().equals(context.getProperty(SOLR_TYPE).getValue())) { - return new HttpSolrClient(solrLocation, httpClient); + return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build(); } else { + // CloudSolrClient.Builder now requires a List of ZK addresses and znode for solr as separate parameters + final String zk[] = solrLocation.split("/"); + final List zkList = Arrays.asList(zk[0].split(",")); + String zkRoot = "/"; + if (zk.length > 1 && ! zk[1].isEmpty()) { + zkRoot += zk[1]; + } + final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue(); final Integer zkClientTimeout = context.getProperty(ZK_CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); final Integer zkConnectionTimeout = context.getProperty(ZK_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); - CloudSolrClient cloudSolrClient = new CloudSolrClient(solrLocation, httpClient); + CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(zkList, Optional.of(zkRoot)).withHttpClient(httpClient).build(); cloudSolrClient.setDefaultCollection(collection); cloudSolrClient.setZkClientTimeout(zkClientTimeout); cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout); diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java index cede9a57f8..9ed5594cc2 100644 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java @@ -30,6 +30,7 @@ import org.apache.nifi.util.TestRunners; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.common.SolrInputDocument; import org.junit.AfterClass; @@ -82,7 +83,8 @@ public class QuerySolrIT { public static void setup() throws IOException, SolrServerException { CloudSolrClient solrClient = createSolrClient(); Path currentDir = Paths.get(ZK_CONFIG_PATH); - solrClient.uploadConfig(currentDir, ZK_CONFIG_NAME); + ZkClientClusterStateProvider stateProvider = new ZkClientClusterStateProvider(SOLR_LOCATION); + stateProvider.uploadConfig(currentDir, ZK_CONFIG_NAME); solrClient.setDefaultCollection(SOLR_COLLECTION); if (!solrClient.getZkStateReader().getClusterState().hasCollection(SOLR_COLLECTION)) { diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java index 55d6d25a0d..e20f2baa79 100644 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java @@ -355,7 +355,7 @@ public class TestGetSolr { runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 1); runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key()); - String expectedXml = "doc1"; + String expectedXml = "doc1"; assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(GetSolr.REL_SUCCESS).get(0))))); } diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java index 0f5fb7e1f4..93f8756df6 100644 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java @@ -406,7 +406,7 @@ public class TestQuerySolr { runner.run(); runner.assertTransferCount(QuerySolr.RESULTS, 1); - String expectedXml = "doc2doc1"; + String expectedXml = "doc2doc1"; assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0))))); solrClient.close(); @@ -425,7 +425,7 @@ public class TestQuerySolr { runner.run(); runner.assertTransferCount(QuerySolr.RESULTS, 1); - String expectedXml = "doc2doc3"; + String expectedXml = "doc2doc3"; assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0))))); solrClient.close(); @@ -447,7 +447,7 @@ public class TestQuerySolr { runner.run(); runner.assertTransferCount(QuerySolr.RESULTS, 1); - String expectedXml = "doc2doc3"; + String expectedXml = "doc2doc3"; assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0))))); solrClient.close(); @@ -471,7 +471,7 @@ public class TestQuerySolr { flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS); flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME); - String expectedXml = "doc1doc0"; + String expectedXml = "doc1doc0"; assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(flowFile)))); solrClient.close(); @@ -492,7 +492,7 @@ public class TestQuerySolr { runner.assertTransferCount(QuerySolr.RESULTS, 1); runner.assertTransferCount(QuerySolr.ORIGINAL, 1); - String expectedXml = "doc0"; + String expectedXml = "doc0"; assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0))))); assertEquals(content, new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.ORIGINAL).get(0)))); @@ -526,9 +526,9 @@ public class TestQuerySolr { startParam += 2; StringBuffer expectedXml = new StringBuffer() - .append("doc") + .append("doc") .append(documentCounter++) - .append("doc") + .append("doc") .append(documentCounter++) .append(""); assertThat(expectedXml.toString(), CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(flowFile)))); diff --git a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/resources/solr/testCollection/conf/solrconfig.xml b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/resources/solr/testCollection/conf/solrconfig.xml index 148a2db3bc..514232d5c5 100644 --- a/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/resources/solr/testCollection/conf/solrconfig.xml +++ b/nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/resources/solr/testCollection/conf/solrconfig.xml @@ -17,4 +17,6 @@ - \ No newline at end of file + + +