NIFI-9641 - Adjusted the extraction of the chroot suffix for solr client connections.

Signed-off-by: Joe Gresock <jgresock@gmail.com>

This closes #5727.
This commit is contained in:
Nathan Gough 2022-01-28 21:55:32 -05:00 committed by Joe Gresock
parent 9bb34188c9
commit dd3c9be847
No known key found for this signature in database
GPG Key ID: 37F5B9B6E258C8B7
3 changed files with 71 additions and 6 deletions

View File

@ -158,6 +158,11 @@
<version>2.6.3</version> <version>2.6.3</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-ssl-context-service</artifactId>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>

View File

@ -236,6 +236,7 @@ public class SolrUtils {
.build(); .build();
public static final String REPEATING_PARAM_PATTERN = "[\\w\\.]+\\.\\d+$"; public static final String REPEATING_PARAM_PATTERN = "[\\w\\.]+\\.\\d+$";
private static final String ROOT_PATH = "/";
public static synchronized SolrClient createSolrClient(final PropertyContext context, final String solrLocation) { public static synchronized SolrClient createSolrClient(final PropertyContext context, final String solrLocation) {
final Integer socketTimeout = context.getProperty(SOLR_SOCKET_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); final Integer socketTimeout = context.getProperty(SOLR_SOCKET_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
@ -281,18 +282,15 @@ public class SolrUtils {
return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build(); return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build();
} else { } else {
// CloudSolrClient.Builder now requires a List of ZK addresses and znode for solr as separate parameters // CloudSolrClient.Builder now requires a List of ZK addresses and znode for solr as separate parameters
final String[] zk = solrLocation.split("/"); final String[] zk = solrLocation.split(ROOT_PATH);
final List zkList = Arrays.asList(zk[0].split(",")); final List zkList = Arrays.asList(zk[0].split(","));
String zkRoot = "/"; String zkChrootPath = getZooKeeperChrootPathSuffix(solrLocation);
if (zk.length > 1 && ! zk[1].isEmpty()) {
zkRoot += zk[1];
}
final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue(); final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue();
final Integer zkClientTimeout = context.getProperty(ZK_CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); final Integer zkClientTimeout = context.getProperty(ZK_CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer zkConnectionTimeout = context.getProperty(ZK_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); final Integer zkConnectionTimeout = context.getProperty(ZK_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(zkList, Optional.of(zkRoot)).withHttpClient(httpClient).build(); CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(zkList, Optional.of(zkChrootPath)).withHttpClient(httpClient).build();
cloudSolrClient.setDefaultCollection(collection); cloudSolrClient.setDefaultCollection(collection);
cloudSolrClient.setZkClientTimeout(zkClientTimeout); cloudSolrClient.setZkClientTimeout(zkClientTimeout);
cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout); cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout);
@ -300,6 +298,16 @@ public class SolrUtils {
} }
} }
private static String getZooKeeperChrootPathSuffix(final String solrLocation) {
String[] zkConnectStringAndChrootSuffix = solrLocation.split("(?=/)", 2);
if (zkConnectStringAndChrootSuffix.length > 1) {
final String chrootSuffix = zkConnectStringAndChrootSuffix[1];
return chrootSuffix;
} else {
return ROOT_PATH;
}
}
/** /**
* Writes each SolrDocument to a record. * Writes each SolrDocument to a record.
*/ */

View File

@ -16,17 +16,22 @@
*/ */
package org.apache.nifi.processors.solr; package org.apache.nifi.processors.solr;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.serialization.SimpleRecordSchema; import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.record.MapRecord; import org.apache.nifi.serialization.record.MapRecord;
import org.apache.nifi.serialization.record.Record; import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordField; import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordFieldType; import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.util.MockPropertyValue;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.util.Collections; import java.util.Collections;
@ -34,12 +39,19 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
public class SolrUtilsTest { public class SolrUtilsTest {
@Mock @Mock
private SolrInputDocument inputDocument; private SolrInputDocument inputDocument;
@Mock
private PropertyContext context;
@Test @Test
public void test() throws Exception { public void test() throws Exception {
// given // given
@ -59,4 +71,44 @@ public class SolrUtilsTest {
// then // then
Mockito.verify(inputDocument, Mockito.times(1)).addField("parent_test", bigDecimalValue); Mockito.verify(inputDocument, Mockito.times(1)).addField("parent_test", bigDecimalValue);
} }
@Test
public void testCreateSolrClientWithChrootSuffix() {
when(context.getProperty(SolrUtils.SOLR_SOCKET_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
when(context.getProperty(SolrUtils.SOLR_CONNECTION_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
when(context.getProperty(SolrUtils.SOLR_MAX_CONNECTIONS)).thenReturn(new MockPropertyValue("5"));
when(context.getProperty(SolrUtils.SOLR_MAX_CONNECTIONS_PER_HOST)).thenReturn(new MockPropertyValue("5"));
when(context.getProperty(SolrUtils.SOLR_TYPE)).thenReturn(new MockPropertyValue(SolrUtils.SOLR_TYPE_CLOUD.getValue()));
when(context.getProperty(SolrUtils.SSL_CONTEXT_SERVICE)).thenReturn(new MockPropertyValue(null));
when(context.getProperty(SolrUtils.KERBEROS_CREDENTIALS_SERVICE)).thenReturn(new MockPropertyValue(null));
when(context.getProperty(SolrUtils.KERBEROS_PRINCIPAL)).thenReturn(new MockPropertyValue("kerb_principal"));
when(context.getProperty(SolrUtils.KERBEROS_PASSWORD)).thenReturn(new MockPropertyValue("kerb_password"));
when(context.getProperty(SolrUtils.COLLECTION)).thenReturn(new MockPropertyValue("collection"));
when(context.getProperty(SolrUtils.ZK_CLIENT_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
when(context.getProperty(SolrUtils.ZK_CONNECTION_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
final String solrLocation = "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/solr/UAT/something";
SolrClient testClient = SolrUtils.createSolrClient(context, solrLocation);
assertNotNull(testClient);
}
@Test
public void testCreateSolrClientWithoutChrootSuffix() {
when(context.getProperty(SolrUtils.SOLR_SOCKET_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
when(context.getProperty(SolrUtils.SOLR_CONNECTION_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
when(context.getProperty(SolrUtils.SOLR_MAX_CONNECTIONS)).thenReturn(new MockPropertyValue("5"));
when(context.getProperty(SolrUtils.SOLR_MAX_CONNECTIONS_PER_HOST)).thenReturn(new MockPropertyValue("5"));
when(context.getProperty(SolrUtils.SOLR_TYPE)).thenReturn(new MockPropertyValue(SolrUtils.SOLR_TYPE_CLOUD.getValue()));
when(context.getProperty(SolrUtils.SSL_CONTEXT_SERVICE)).thenReturn(new MockPropertyValue(null));
when(context.getProperty(SolrUtils.KERBEROS_CREDENTIALS_SERVICE)).thenReturn(new MockPropertyValue(null));
when(context.getProperty(SolrUtils.KERBEROS_PRINCIPAL)).thenReturn(new MockPropertyValue("kerb_principal"));
when(context.getProperty(SolrUtils.KERBEROS_PASSWORD)).thenReturn(new MockPropertyValue("kerb_password"));
when(context.getProperty(SolrUtils.COLLECTION)).thenReturn(new MockPropertyValue("collection"));
when(context.getProperty(SolrUtils.ZK_CLIENT_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
when(context.getProperty(SolrUtils.ZK_CONNECTION_TIMEOUT)).thenReturn(new MockPropertyValue("2 s"));
final String solrLocation = "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181";
SolrClient testClient = SolrUtils.createSolrClient(context, solrLocation);
assertNotNull(testClient);
}
} }