HADOOP-15993. Upgrade Kafka to 2.4.0 in hadoop-kafka module. (#1796)

LICENSE-binary

@@ -283,7 +283,6 @@ io.swagger:swagger-annotations:1.5.4
 javax.inject:javax.inject:1
 log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
-net.jpountz.lz4:lz4:1.2.0
 net.minidev:accessors-smart:1.2
 net.minidev:json-smart:2.3
 org.apache.avro:avro:1.7.7
@@ -308,7 +307,7 @@ org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
-org.apache.kafka:kafka-clients:0.8.2.1
+org.apache.kafka:kafka-clients:2.4.0
 org.apache.kerby:kerb-admin:1.0.1
 org.apache.kerby:kerb-client:1.0.1
 org.apache.kerby:kerb-common:1.0.1
@@ -346,6 +345,7 @@ org.eclipse.jetty:jetty-xml:9.3.27.v20190418
 org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418
 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418
 org.ehcache:ehcache:3.3.1
+org.lz4:lz4-java:1.6.0
 org.objenesis:objenesis:1.0:compile
 org.xerial.snappy:snappy-java:1.0.5
 org.yaml:snakeyaml:1.16:
@@ -365,6 +365,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
+com.github.luben:zstd-jni:1.4.3-1
 dnsjava:dnsjava:2.1.7
 org.codehaus.woodstox:stax2-api:3.1.4
 

hadoop-project/pom.xml

@@ -48,7 +48,7 @@
     <!-- Version number for xerces used by JDiff -->
     <xerces.jdiff.version>2.11.0</xerces.jdiff.version>
 
-    <kafka.version>0.8.2.1</kafka.version>
+    <kafka.version>2.4.0</kafka.version>
 
     <hadoop.assemblies.version>3.3.0-SNAPSHOT</hadoop.assemblies.version>
     <commons-daemon.version>1.0.13</commons-daemon.version>

licenses-binary/LICENSE-zstd-jni.txt

@@ -0,0 +1,26 @@
+Zstd-jni: JNI bindings to Zstd Library
+
+Copyright (c) 2015-present, Luben Karavelov/ All rights reserved.
+
+BSD License
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.