NIFI-7030 Add Kerberos principal and password properties to Solr processors

Updating validation logic to be consistent with other password-based kerberos processors, removing getPassword from KerberosPasswordUser as it was only used from testing

This closes #4062.
This commit is contained in:
Bryan Bende 2020-02-18 14:54:27 -05:00 committed by jstorck
parent e0fc75a963
commit 852d2f3210
7 changed files with 204 additions and 44 deletions

View File

@ -18,24 +18,6 @@
*/
package org.apache.nifi.processors.solr;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.Stateful;
@ -64,7 +46,6 @@ import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
import org.apache.nifi.util.StopWatch;
import org.apache.nifi.util.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.QueryRequest;
@ -73,20 +54,39 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CursorMarkParams;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_CREDENTIALS_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_TYPE;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_USERNAME;
import static org.apache.nifi.processors.solr.SolrUtils.COLLECTION;
import static org.apache.nifi.processors.solr.SolrUtils.SSL_CONTEXT_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_SOCKET_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_CREDENTIALS_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PRINCIPAL;
import static org.apache.nifi.processors.solr.SolrUtils.RECORD_WRITER;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_CONNECTION_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_LOCATION;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_MAX_CONNECTIONS;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_MAX_CONNECTIONS_PER_HOST;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_SOCKET_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_TYPE;
import static org.apache.nifi.processors.solr.SolrUtils.SSL_CONTEXT_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.ZK_CLIENT_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.ZK_CONNECTION_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_LOCATION;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_USERNAME;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.RECORD_WRITER;
@Tags({"Apache", "Solr", "Get", "Pull", "Records"})
@InputRequirement(Requirement.INPUT_FORBIDDEN)
@ -182,6 +182,8 @@ public class GetSolr extends SolrProcessor {
descriptors.add(RETURN_FIELDS);
descriptors.add(BATCH_SIZE);
descriptors.add(KERBEROS_CREDENTIALS_SERVICE);
descriptors.add(KERBEROS_PRINCIPAL);
descriptors.add(KERBEROS_PASSWORD);
descriptors.add(BASIC_USERNAME);
descriptors.add(BASIC_PASSWORD);
descriptors.add(SSL_CONTEXT_SERVICE);

View File

@ -58,6 +58,8 @@ import static org.apache.nifi.processors.solr.SolrUtils.BASIC_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_USERNAME;
import static org.apache.nifi.processors.solr.SolrUtils.COLLECTION;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_CREDENTIALS_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PRINCIPAL;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_CONNECTION_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_LOCATION;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_MAX_CONNECTIONS;
@ -136,6 +138,8 @@ public class PutSolrContentStream extends SolrProcessor {
descriptors.add(CONTENT_TYPE);
descriptors.add(COMMIT_WITHIN);
descriptors.add(KERBEROS_CREDENTIALS_SERVICE);
descriptors.add(KERBEROS_PRINCIPAL);
descriptors.add(KERBEROS_PASSWORD);
descriptors.add(BASIC_USERNAME);
descriptors.add(BASIC_PASSWORD);
descriptors.add(SSL_CONTEXT_SERVICE);

View File

@ -64,6 +64,8 @@ import static org.apache.nifi.processors.solr.SolrUtils.BASIC_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_USERNAME;
import static org.apache.nifi.processors.solr.SolrUtils.COLLECTION;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_CREDENTIALS_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PRINCIPAL;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_CONNECTION_TIMEOUT;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_LOCATION;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_MAX_CONNECTIONS;
@ -164,6 +166,8 @@ public class PutSolrRecord extends SolrProcessor {
descriptors.add(FIELDS_TO_INDEX);
descriptors.add(COMMIT_WITHIN);
descriptors.add(KERBEROS_CREDENTIALS_SERVICE);
descriptors.add(KERBEROS_PRINCIPAL);
descriptors.add(KERBEROS_PASSWORD);
descriptors.add(BASIC_USERNAME);
descriptors.add(BASIC_PASSWORD);
descriptors.add(SSL_CONTEXT_SERVICE);

View File

@ -73,6 +73,8 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_CREDENTIALS_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PRINCIPAL;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_TYPE;
import static org.apache.nifi.processors.solr.SolrUtils.COLLECTION;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_TYPE_CLOUD;
@ -260,6 +262,8 @@ public class QuerySolr extends SolrProcessor {
descriptors.add(SOLR_PARAM_ROWS);
descriptors.add(AMOUNT_DOCUMENTS_TO_RETURN);
descriptors.add(KERBEROS_CREDENTIALS_SERVICE);
descriptors.add(KERBEROS_PRINCIPAL);
descriptors.add(KERBEROS_PASSWORD);
descriptors.add(BASIC_USERNAME);
descriptors.add(BASIC_PASSWORD);
descriptors.add(SSL_CONTEXT_SERVICE);

View File

@ -29,6 +29,7 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.security.krb.KerberosAction;
import org.apache.nifi.security.krb.KerberosPasswordUser;
import org.apache.nifi.security.krb.KerberosUser;
import org.apache.nifi.security.krb.KerberosKeytabUser;
import org.apache.nifi.ssl.SSLContextService;
@ -45,6 +46,8 @@ import static org.apache.nifi.processors.solr.SolrUtils.BASIC_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.BASIC_USERNAME;
import static org.apache.nifi.processors.solr.SolrUtils.COLLECTION;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_CREDENTIALS_SERVICE;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PASSWORD;
import static org.apache.nifi.processors.solr.SolrUtils.KERBEROS_PRINCIPAL;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_LOCATION;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_TYPE;
import static org.apache.nifi.processors.solr.SolrUtils.SOLR_TYPE_CLOUD;
@ -76,16 +79,26 @@ public abstract class SolrProcessor extends AbstractProcessor {
this.solrClient = createSolrClient(context, solrLocation);
final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).evaluateAttributeExpressions().getValue();
final String kerberosPassword = context.getProperty(KERBEROS_PASSWORD).getValue();
final KerberosCredentialsService kerberosCredentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
if (kerberosCredentialsService != null) {
this.kerberosUser = createKeytabUser(kerberosCredentialsService);
this.kerberosUser = createKerberosKeytabUser(kerberosCredentialsService);
} else if (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword)) {
this.kerberosUser = createKerberosPasswordUser(kerberosPrincipal, kerberosPassword);
}
}
protected KerberosUser createKeytabUser(final KerberosCredentialsService kerberosCredentialsService) {
protected KerberosUser createKerberosKeytabUser(final KerberosCredentialsService kerberosCredentialsService) {
return new KerberosKeytabUser(kerberosCredentialsService.getPrincipal(), kerberosCredentialsService.getKeytab());
}
protected KerberosUser createKerberosPasswordUser(final String principal, final String password) {
return new KerberosPasswordUser(principal, password);
}
@OnStopped
public final void closeClient() {
if (solrClient != null) {
@ -108,7 +121,7 @@ public abstract class SolrProcessor extends AbstractProcessor {
@Override
public final void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
final KerberosUser kerberosUser = getKerberosKeytabUser();
final KerberosUser kerberosUser = getKerberosUser();
if (kerberosUser == null) {
doOnTrigger(context, session);
} else {
@ -173,7 +186,7 @@ public abstract class SolrProcessor extends AbstractProcessor {
return basicAuthEnabled;
}
protected final KerberosUser getKerberosKeytabUser() {
protected final KerberosUser getKerberosUser() {
return kerberosUser;
}
@ -214,7 +227,7 @@ public abstract class SolrProcessor extends AbstractProcessor {
}
}
// Validate that we username and password are provided together, or that neither are provided
// Validate that username and password are provided together, or that neither are provided
final String username = context.getProperty(BASIC_USERNAME).evaluateAttributeExpressions().getValue();
final String password = context.getProperty(BASIC_PASSWORD).evaluateAttributeExpressions().getValue();
@ -237,13 +250,53 @@ public abstract class SolrProcessor extends AbstractProcessor {
.build());
}
// Validate that only kerberos or basic auth can be set, but not both
// Validate that kerberos principal and password are provided together, or that neither are provided
final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).evaluateAttributeExpressions().getValue();
final String kerberosPassword = context.getProperty(KERBEROS_PASSWORD).evaluateAttributeExpressions().getValue();
final boolean kerberosPrincipalProvided = !StringUtils.isBlank(kerberosPrincipal);
final boolean kerberosPasswordProvided = !StringUtils.isBlank(kerberosPassword);
if (kerberosPrincipalProvided && !kerberosPasswordProvided) {
problems.add(new ValidationResult.Builder()
.subject(KERBEROS_PASSWORD.getDisplayName())
.valid(false)
.explanation("a password must be provided for the given principal")
.build());
}
if (kerberosPasswordProvided && !kerberosPrincipalProvided) {
problems.add(new ValidationResult.Builder()
.subject(KERBEROS_PRINCIPAL.getDisplayName())
.valid(false)
.explanation("a principal must be provided for the given password")
.build());
}
// Validate that only one of kerberos princpal/password, kerberos creds service, or basic auth can be set
final KerberosCredentialsService kerberosCredentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
if (kerberosCredentialsService != null && basicUsernameProvided && basicPasswordProvided) {
problems.add(new ValidationResult.Builder()
.subject(KERBEROS_CREDENTIALS_SERVICE.getDisplayName())
.valid(false)
.explanation("basic auth and kerberos cannot be configured at the same time")
.explanation("basic auth and kerberos credential service cannot be configured at the same time")
.build());
}
if (kerberosCredentialsService != null && (kerberosPrincipalProvided || kerberosPasswordProvided)) {
problems.add(new ValidationResult.Builder()
.subject(KERBEROS_CREDENTIALS_SERVICE.getDisplayName())
.valid(false)
.explanation("kerberos principal/password and kerberos credential service cannot be configured at the same time")
.build());
}
if (kerberosPrincipalProvided && kerberosPasswordProvided && basicUsernameProvided && basicPasswordProvided) {
problems.add(new ValidationResult.Builder()
.subject(KERBEROS_PRINCIPAL.getDisplayName())
.valid(false)
.explanation("basic auth and kerberos principal/password cannot be configured at the same time")
.build());
}

View File

@ -122,8 +122,9 @@ public class SolrUtils {
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
public static final PropertyDescriptor BASIC_USERNAME = new PropertyDescriptor
.Builder().name("Username")
public static final PropertyDescriptor BASIC_USERNAME = new PropertyDescriptor.Builder()
.name("Username")
.displayName("Basic Auth Username")
.description("The username to use when Solr is configured with basic authentication.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
@ -131,8 +132,9 @@ public class SolrUtils {
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
public static final PropertyDescriptor BASIC_PASSWORD = new PropertyDescriptor
.Builder().name("Password")
public static final PropertyDescriptor BASIC_PASSWORD = new PropertyDescriptor.Builder()
.name("Password")
.displayName("Basic Auth Password")
.description("The password to use when Solr is configured with basic authentication.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
@ -149,6 +151,26 @@ public class SolrUtils {
.required(false)
.build();
public static final PropertyDescriptor KERBEROS_PRINCIPAL = new PropertyDescriptor.Builder()
.name("kerberos-principal")
.displayName("Kerberos Principal")
.description("The principal to use when specifying the principal and password directly in the processor for authenticating to Solr via Kerberos.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING))
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
public static final PropertyDescriptor KERBEROS_PASSWORD = new PropertyDescriptor.Builder()
.name("kerberos-password")
.displayName("Kerberos Password")
.description("The password to use when specifying the principal and password directly in the processor for authenticating to Solr via Kerberos.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING))
.sensitive(true)
.build();
public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
.name("SSL Context Service")
.description("The Controller Service to use in order to obtain an SSL Context. This property must be set when communicating with a Solr over https.")
@ -213,6 +235,8 @@ public class SolrUtils {
final Integer maxConnectionsPerHost = context.getProperty(SOLR_MAX_CONNECTIONS_PER_HOST).asInteger();
final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
final KerberosCredentialsService kerberosCredentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).evaluateAttributeExpressions().getValue();
final String kerberosPassword = context.getProperty(KERBEROS_PASSWORD).getValue();
final ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
@ -221,7 +245,7 @@ public class SolrUtils {
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
// has to happen before the client is created below so that correct configurer would be set if needed
if (kerberosCredentialsService != null) {
if (kerberosCredentialsService != null || (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword))) {
HttpClientUtil.setConfigurer(new KerberosHttpClientConfigurer());
}

View File

@ -429,7 +429,7 @@ public class TestPutSolrContentStream {
}
@Test
public void testBasicAuthAndKerberosNotAllowedTogether() throws IOException, InitializationException {
public void testBasicAuthAndKerberosCredentialServiceNotAllowedTogether() throws IOException, InitializationException {
final SolrClient solrClient = createEmbeddedSolrClient(DEFAULT_SOLR_CORE);
final TestableProcessor proc = new TestableProcessor(solrClient);
final TestRunner runner = createDefaultTestRunner(proc);
@ -453,7 +453,67 @@ public class TestPutSolrContentStream {
runner.assertValid();
proc.onScheduled(runner.getProcessContext());
final KerberosUser kerberosUser = proc.getMockKerberosKeytabUser();;
final KerberosUser kerberosUser = proc.getMockKerberosKeytabUser();
Assert.assertNotNull(kerberosUser);
Assert.assertEquals(principal, kerberosUser.getPrincipal());
Assert.assertEquals(keytab, ((KerberosKeytabUser)kerberosUser).getKeytabFile());
}
@Test
public void testBasicAuthAndKerberosPrincipalPasswordNotAllowedTogether() throws IOException, InitializationException {
final SolrClient solrClient = createEmbeddedSolrClient(DEFAULT_SOLR_CORE);
final TestableProcessor proc = new TestableProcessor(solrClient);
final TestRunner runner = createDefaultTestRunner(proc);
runner.assertValid();
runner.setProperty(SolrUtils.BASIC_USERNAME, "user1");
runner.setProperty(SolrUtils.BASIC_PASSWORD, "password");
runner.assertValid();
final String kerberosPrincipal = "nifi@FOO.COM";
final String kerberosPassword = "nifi";
runner.setProperty(SolrUtils.KERBEROS_PRINCIPAL, kerberosPrincipal);
runner.setProperty(SolrUtils.KERBEROS_PASSWORD, kerberosPassword);
runner.assertNotValid();
runner.removeProperty(SolrUtils.BASIC_USERNAME);
runner.removeProperty(SolrUtils.BASIC_PASSWORD);
runner.assertValid();
proc.onScheduled(runner.getProcessContext());
final KerberosUser kerberosUser = proc.getMockKerberosKeytabUser();
Assert.assertNotNull(kerberosUser);
Assert.assertEquals(kerberosPrincipal, kerberosUser.getPrincipal());
}
@Test
public void testKerberosPrincipalPasswordAndKerberosCredentialServiceNotAllowedTogether() throws IOException, InitializationException {
final SolrClient solrClient = createEmbeddedSolrClient(DEFAULT_SOLR_CORE);
final TestableProcessor proc = new TestableProcessor(solrClient);
final TestRunner runner = createDefaultTestRunner(proc);
runner.assertValid();
final String kerberosPrincipal = "nifi@FOO.COM";
final String kerberosPassword = "nifi";
runner.setProperty(SolrUtils.KERBEROS_PRINCIPAL, kerberosPrincipal);
runner.setProperty(SolrUtils.KERBEROS_PASSWORD, kerberosPassword);
final String principal = "nifi@FOO.COM";
final String keytab = "src/test/resources/foo.keytab";
final KerberosCredentialsService kerberosCredentialsService = new MockKerberosCredentialsService(principal, keytab);
runner.addControllerService("kerb-credentials", kerberosCredentialsService);
runner.enableControllerService(kerberosCredentialsService);
runner.setProperty(SolrUtils.KERBEROS_CREDENTIALS_SERVICE, "kerb-credentials");
runner.assertNotValid();
runner.removeProperty(SolrUtils.KERBEROS_PRINCIPAL);
runner.removeProperty(SolrUtils.KERBEROS_PASSWORD);
runner.assertValid();
proc.onScheduled(runner.getProcessContext());
final KerberosUser kerberosUser = proc.getMockKerberosKeytabUser();
Assert.assertNotNull(kerberosUser);
Assert.assertEquals(principal, kerberosUser.getPrincipal());
Assert.assertEquals(keytab, ((KerberosKeytabUser)kerberosUser).getKeytabFile());
@ -666,16 +726,25 @@ public class TestPutSolrContentStream {
}
@Override
protected KerberosUser createKeytabUser(KerberosCredentialsService kerberosCredentialsService) {
protected KerberosUser createKerberosKeytabUser(KerberosCredentialsService kerberosCredentialsService) {
if (kerberosUser != null) {
return kerberosUser;
} else {
return super.createKeytabUser(kerberosCredentialsService);
return super.createKerberosKeytabUser(kerberosCredentialsService);
}
}
@Override
protected KerberosUser createKerberosPasswordUser(String principal, String password) {
if (kerberosUser != null) {
return kerberosUser;
} else {
return super.createKerberosPasswordUser(principal, password);
}
}
public KerberosUser getMockKerberosKeytabUser() {
return super.getKerberosKeytabUser();
return super.getKerberosUser();
}
}