From e0e4b3407a3b90d29858fdc0c73ccccd8862abc7 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Thu, 18 Aug 2016 11:09:11 -0400 Subject: [PATCH] NIFI-2574 This clsoes #887. Fixing NPE when using kerberos keytab location from contexts, and cleaning up hadoop/hbase/hive kerberos variables --- .../StandardReportingInitializationContext.java | 2 +- ...tandardControllerServiceInitializationContext.java | 2 +- .../StandardProcessorInitializationContext.java | 2 +- .../processors/hadoop/AbstractHadoopProcessor.java | 11 ++++------- .../processors/hadoop/GetHDFSSequenceFileTest.java | 3 ++- .../apache/nifi/processors/hadoop/GetHDFSTest.java | 3 ++- .../apache/nifi/processors/hadoop/PutHDFSTest.java | 4 ++-- .../nifi/processors/hadoop/SimpleHadoopProcessor.java | 4 +++- .../hadoop/TestCreateHadoopSequenceFile.java | 2 +- .../apache/nifi/processors/hadoop/TestFetchHDFS.java | 3 ++- .../apache/nifi/processors/hadoop/TestListHDFS.java | 2 +- .../processors/hadoop/inotify/TestGetHDFSEvents.java | 3 ++- .../org/apache/nifi/dbcp/hive/HiveConnectionPool.java | 4 ---- .../apache/nifi/processors/hive/PutHiveStreaming.java | 4 ---- .../apache/nifi/hbase/HBase_1_1_2_ClientService.java | 10 +++------- .../nifi/hbase/TestHBase_1_1_2_ClientService.java | 2 +- 16 files changed, 26 insertions(+), 35 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingInitializationContext.java index a05b5cf8cb..d96d2b7000 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingInitializationContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingInitializationContext.java @@ -127,7 +127,7 @@ public class StandardReportingInitializationContext implements ReportingInitiali @Override public File getKerberosServiceKeytab() { - return new File(nifiProperties.getKerberosServiceKeytabLocation()); + return nifiProperties.getKerberosServiceKeytabLocation() == null ? null : new File(nifiProperties.getKerberosServiceKeytabLocation()); } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInitializationContext.java index 0fe51bcb8e..f169662a1e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInitializationContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInitializationContext.java @@ -102,7 +102,7 @@ public class StandardControllerServiceInitializationContext implements Controlle @Override public File getKerberosServiceKeytab() { - return new File(nifiProperties.getKerberosServiceKeytabLocation()); + return nifiProperties.getKerberosServiceKeytabLocation() == null ? null : new File(nifiProperties.getKerberosServiceKeytabLocation()); } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessorInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessorInitializationContext.java index 0449655eb3..a0300eef9e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessorInitializationContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessorInitializationContext.java @@ -69,7 +69,7 @@ public class StandardProcessorInitializationContext implements ProcessorInitiali @Override public File getKerberosServiceKeytab() { - return new File(nifiProperties.getKerberosServiceKeytabLocation()); + return nifiProperties.getKerberosServiceKeytabLocation() == null ? null : new File(nifiProperties.getKerberosServiceKeytabLocation()); } @Override diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java index 09450c2c0a..ecb24aaeee 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java @@ -127,9 +127,7 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor { private long lastKerberosReloginTime; protected KerberosProperties kerberosProperties; protected List properties; - private volatile String kerberosServicePrincipal = null; private volatile File kerberosConfigFile = null; - private volatile File kerberosServiceKeytab = null; // variables shared by all threads of this processor // Hadoop Configuration, Filesystem, and UserGroupInformation (optional) @@ -141,7 +139,9 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor { @Override protected void init(ProcessorInitializationContext context) { hdfsResources.set(new HdfsResources(null, null, null)); - kerberosProperties = getKerberosProperties(); + + kerberosConfigFile = context.getKerberosConfigurationFile(); + kerberosProperties = getKerberosProperties(kerberosConfigFile); List props = new ArrayList<>(); props.add(HADOOP_CONFIGURATION_RESOURCES); @@ -149,12 +149,9 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor { props.add(kerberosProperties.getKerberosKeytab()); props.add(KERBEROS_RELOGIN_PERIOD); properties = Collections.unmodifiableList(props); - kerberosServicePrincipal = context.getKerberosServicePrincipal(); - kerberosConfigFile = context.getKerberosConfigurationFile(); - kerberosServiceKeytab = context.getKerberosServiceKeytab(); } - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return new KerberosProperties(kerberosConfigFile); } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSSequenceFileTest.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSSequenceFileTest.java index 691619543d..c5a1c021a7 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSSequenceFileTest.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSSequenceFileTest.java @@ -30,6 +30,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; +import java.io.File; import java.io.IOException; import java.security.PrivilegedExceptionAction; @@ -113,7 +114,7 @@ public class GetHDFSSequenceFileTest { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return kerberosProperties; } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSTest.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSTest.java index 13c5436df2..325ba3a862 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSTest.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/GetHDFSTest.java @@ -30,6 +30,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Collection; @@ -246,7 +247,7 @@ public class GetHDFSTest { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProperties; } } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java index 72c7da991e..482ba69718 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java @@ -273,7 +273,7 @@ public class PutHDFSTest { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProperties; } }); @@ -375,7 +375,7 @@ public class PutHDFSTest { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProperties; } } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/SimpleHadoopProcessor.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/SimpleHadoopProcessor.java index 4d382ebc60..4aa8e54678 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/SimpleHadoopProcessor.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/SimpleHadoopProcessor.java @@ -21,6 +21,8 @@ import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.exception.ProcessException; +import java.io.File; + public class SimpleHadoopProcessor extends AbstractHadoopProcessor { private KerberosProperties testKerberosProperties; @@ -34,7 +36,7 @@ public class SimpleHadoopProcessor extends AbstractHadoopProcessor { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProperties; } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java index af5fb3b017..419323f2e4 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java @@ -213,7 +213,7 @@ public class TestCreateHadoopSequenceFile { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerbersProperties; } } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java index bbfb0953d8..6153980759 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestFetchHDFS.java @@ -23,6 +23,7 @@ import org.apache.nifi.util.TestRunners; import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -111,7 +112,7 @@ public class TestFetchHDFS { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProps; } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestListHDFS.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestListHDFS.java index a76557241f..5f540994c2 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestListHDFS.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestListHDFS.java @@ -308,7 +308,7 @@ public class TestListHDFS { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProps; } diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/inotify/TestGetHDFSEvents.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/inotify/TestGetHDFSEvents.java index 939cbfc511..ef9e9c618e 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/inotify/TestGetHDFSEvents.java +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/inotify/TestGetHDFSEvents.java @@ -34,6 +34,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.File; import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -252,7 +253,7 @@ public class TestGetHDFSEvents { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return testKerberosProperties; } diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java index 0f79002228..9006d41ace 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java @@ -133,9 +133,7 @@ public class HiveConnectionPool extends AbstractControllerService implements Hiv private volatile HiveConfigurator hiveConfigurator = new HiveConfigurator(); private volatile UserGroupInformation ugi; - private volatile String kerberosServicePrincipal = null; private volatile File kerberosConfigFile = null; - private volatile File kerberosServiceKeytab = null; private volatile KerberosProperties kerberosProperties; static { @@ -151,9 +149,7 @@ public class HiveConnectionPool extends AbstractControllerService implements Hiv @Override protected void init(final ControllerServiceInitializationContext context) { - kerberosServicePrincipal = context.getKerberosServicePrincipal(); kerberosConfigFile = context.getKerberosConfigurationFile(); - kerberosServiceKeytab = context.getKerberosServiceKeytab(); kerberosProperties = new KerberosProperties(kerberosConfigFile); properties.add(kerberosProperties.getKerberosPrincipal()); properties.add(kerberosProperties.getKerberosKeytab()); diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java index 14eeb2a797..b4822b2614 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java @@ -244,9 +244,7 @@ public class PutHiveStreaming extends AbstractProcessor { private static final long TICKET_RENEWAL_PERIOD = 60000; protected KerberosProperties kerberosProperties; - private volatile String kerberosServicePrincipal = null; private volatile File kerberosConfigFile = null; - private volatile File kerberosServiceKeytab = null; protected volatile HiveConfigurator hiveConfigurator = new HiveConfigurator(); protected volatile UserGroupInformation ugi; @@ -285,9 +283,7 @@ public class PutHiveStreaming extends AbstractProcessor { @Override protected void init(ProcessorInitializationContext context) { - kerberosServicePrincipal = context.getKerberosServicePrincipal(); kerberosConfigFile = context.getKerberosConfigurationFile(); - kerberosServiceKeytab = context.getKerberosServiceKeytab(); kerberosProperties = new KerberosProperties(kerberosConfigFile); propertyDescriptors.add(kerberosProperties.getKerberosPrincipal()); propertyDescriptors.add(kerberosProperties.getKerberosKeytab()); diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java index e91c32f487..5517474237 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java @@ -91,16 +91,15 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme private List properties; private KerberosProperties kerberosProperties; - private volatile String kerberosServicePrincipal = null; private volatile File kerberosConfigFile = null; - private volatile File kerberosServiceKeytab = null; // Holder of cached Configuration information so validation does not reload the same config over and over private final AtomicReference validationResourceHolder = new AtomicReference<>(); @Override protected void init(ControllerServiceInitializationContext config) throws InitializationException { - this.kerberosProperties = getKerberosProperties(); + kerberosConfigFile = config.getKerberosConfigurationFile(); + kerberosProperties = getKerberosProperties(kerberosConfigFile); List props = new ArrayList<>(); props.add(HADOOP_CONF_FILES); @@ -111,12 +110,9 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme props.add(ZOOKEEPER_ZNODE_PARENT); props.add(HBASE_CLIENT_RETRIES); this.properties = Collections.unmodifiableList(props); - kerberosServicePrincipal = config.getKerberosServicePrincipal(); - kerberosConfigFile = config.getKerberosConfigurationFile(); - kerberosServiceKeytab = config.getKerberosServiceKeytab(); } - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return new KerberosProperties(kerberosConfigFile); } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java index 33536b5be9..469033d2b5 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java @@ -451,7 +451,7 @@ public class TestHBase_1_1_2_ClientService { } @Override - protected KerberosProperties getKerberosProperties() { + protected KerberosProperties getKerberosProperties(File kerberosConfigFile) { return kerberosProperties; }