[Issue #6967] NoClassDefFoundError when using druid-hdfs-storage (#7015)

* Fix:
  1. hadoop-common dependency for druid-hdfs and druid-kerberos extensions
 Refactoring:
  2. Hadoop config call in the inner static class to avoid class path conflicts for stopGracefully kill

* Fix:
  1. hadoop-common test dependency

* Fix:
  1. Avoid issue of kill command once the job is actually completed

extensions-core/druid-kerberos/pom.xml

@@ -71,6 +71,7 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.compile.version}</version>
+      <scope>compile</scope>
       <exclusions>
         <exclusion>
           <groupId>commons-cli</groupId>

extensions-core/hdfs-storage/pom.xml

@@ -151,6 +151,130 @@
               </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.compile.version}</version>
+          <scope>compile</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>commons-cli</groupId>
+              <artifactId>commons-cli</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-httpclient</groupId>
+              <artifactId>commons-httpclient</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>log4j</groupId>
+              <artifactId>log4j</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-codec</groupId>
+              <artifactId>commons-codec</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-logging</groupId>
+              <artifactId>commons-logging</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-io</groupId>
+              <artifactId>commons-io</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-lang</groupId>
+              <artifactId>commons-lang</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.httpcomponents</groupId>
+              <artifactId>httpclient</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.httpcomponents</groupId>
+              <artifactId>httpcore</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.codehaus.jackson</groupId>
+              <artifactId>jackson-core-asl</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.codehaus.jackson</groupId>
+              <artifactId>jackson-mapper-asl</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.zookeeper</groupId>
+              <artifactId>zookeeper</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.slf4j</groupId>
+              <artifactId>slf4j-api</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.slf4j</groupId>
+              <artifactId>slf4j-log4j12</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>javax.ws.rs</groupId>
+              <artifactId>jsr311-api</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.code.findbugs</groupId>
+              <artifactId>jsr305</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.mortbay.jetty</groupId>
+              <artifactId>jetty-util</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.hadoop</groupId>
+              <artifactId>hadoop-annotations</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.protobuf</groupId>
+              <artifactId>protobuf-java</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.sun.jersey</groupId>
+              <artifactId>jersey-core</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.curator</groupId>
+              <artifactId>curator-client</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.commons</groupId>
+              <artifactId>commons-math3</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.avro</groupId>
+              <artifactId>avro</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>net.java.dev.jets3t</groupId>
+              <artifactId>jets3t</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.sun.jersey</groupId>
+              <artifactId>jersey-json</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.jcraft</groupId>
+              <artifactId>jsch</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.mortbay.jetty</groupId>
+              <artifactId>jetty</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.sun.jersey</groupId>
+              <artifactId>jersey-server</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-aws</artifactId>
@@ -164,6 +288,13 @@
         </dependency>
 
         <!-- Tests -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.compile.version}</version>
+          <classifier>tests</classifier>
+          <scope>test</scope>
+        </dependency>
         <dependency>
           <groupId>junit</groupId>
           <artifactId>junit</artifactId>
@@ -189,13 +320,6 @@
           <classifier>tests</classifier>
           <scope>test</scope>
         </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-          <version>${hadoop.compile.version}</version>
-          <classifier>tests</classifier>
-          <scope>test</scope>
-        </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-hdfs</artifactId>

indexing-hadoop/pom.xml

@@ -83,6 +83,22 @@
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
         <!-- Tests -->
         <dependency>
@@ -130,22 +146,6 @@
             <version>${hadoop.compile.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-mapreduce-client-core</artifactId>
-            <scope>provided</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.apache.druid</groupId>
             <artifactId>druid-server</artifactId>

indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java

@@ -430,21 +430,12 @@ public class HadoopIndexTask extends HadoopTask implements ChatHandler
   @Override
   public void stopGracefully(TaskConfig taskConfig)
   {
-    final ClassLoader oldLoader = Thread.currentThread().getContextClassLoader();
-    File hadoopJobIdFile = new File(getHadoopJobIdFileName());
-    String jobId = null;
+    // To avoid issue of kill command once the ingestion task is actually completed
+    if (!ingestionState.equals(IngestionState.COMPLETED)) {
+      final ClassLoader oldLoader = Thread.currentThread().getContextClassLoader();
+      String hadoopJobIdFile = getHadoopJobIdFileName();
 
-    try {
-      if (hadoopJobIdFile.exists()) {
-        jobId = HadoopDruidIndexerConfig.JSON_MAPPER.readValue(hadoopJobIdFile, String.class);
-      }
-    }
-    catch (Exception e) {
-      log.warn(e, "exeption while reading Hadoop Job ID from: %s", hadoopJobIdFile);
-    }
-
-    try {
-      if (jobId != null) {
+      try {
         ClassLoader loader = HadoopTask.buildClassLoader(getHadoopDependencyCoordinates(),
             taskConfig.getDefaultHadoopCoordinates());
 
@@ -452,28 +443,28 @@ public class HadoopIndexTask extends HadoopTask implements ChatHandler
             "org.apache.druid.indexing.common.task.HadoopIndexTask$HadoopKillMRJobIdProcessingRunner",
             loader
         );
+
         String[] buildKillJobInput = new String[]{
-            "-kill",
-            jobId
+            hadoopJobIdFile
         };
 
         Class<?> buildKillJobRunnerClass = killMRJobInnerProcessingRunner.getClass();
         Method innerProcessingRunTask = buildKillJobRunnerClass.getMethod("runTask", buildKillJobInput.getClass());
 
         Thread.currentThread().setContextClassLoader(loader);
-        final String killStatusString = (String) innerProcessingRunTask.invoke(
+        final String killStatusString[] = (String[]) innerProcessingRunTask.invoke(
             killMRJobInnerProcessingRunner,
             new Object[]{buildKillJobInput}
         );
 
-        log.info(StringUtils.format("Tried killing job %s , status: %s", jobId, killStatusString));
+        log.info(StringUtils.format("Tried killing job: [%s], status: [%s]", killStatusString[0], killStatusString[1]));
+      }
+      catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+      finally {
+        Thread.currentThread().setContextClassLoader(oldLoader);
       }
-    }
-    catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-    finally {
-      Thread.currentThread().setContextClassLoader(oldLoader);
     }
 
   }
@@ -722,10 +713,29 @@ public class HadoopIndexTask extends HadoopTask implements ChatHandler
   @SuppressWarnings("unused")
   public static class HadoopKillMRJobIdProcessingRunner
   {
-    public String runTask(String[] args) throws Exception
+    public String[] runTask(String[] args) throws Exception
     {
-      int res = ToolRunner.run(new JobClient(), args);
-      return res == 0 ? "Success" : "Fail";
+      File hadoopJobIdFile = new File(args[0]);
+      String jobId = null;
+
+      try {
+        if (hadoopJobIdFile.exists()) {
+          jobId = HadoopDruidIndexerConfig.JSON_MAPPER.readValue(hadoopJobIdFile, String.class);
+        }
+      }
+      catch (Exception e) {
+        log.warn(e, "exeption while reading hadoop job id from: [%s]", hadoopJobIdFile);
+      }
+
+      if (jobId != null) {
+        int res = ToolRunner.run(new JobClient(), new String[]{
+            "-kill",
+            jobId
+        });
+
+        return new String[] {jobId, (res == 0 ? "Success" : "Fail")};
+      }
+      return new String[] {jobId, "Fail"};
     }
   }