Upgrading HDFS Repository Plugin to use HDFS 2.8.1 Client (#25497)
Hadoop 2.7.x libraries fail when running on JDK9 due to the version string changing to a single character. On Hadoop 2.8, this is no longer a problem, and it is unclear on whether the fix will be backported to the 2.7 branch. This commit upgrades our dependency of Hadoop for the HDFS Repository to 2.8.1.
This commit is contained in:
James Baiera
GitHub
parent
c70c440050
commit
74f4a14d82
|
|
@ -32,7 +32,7 @@ esplugin {
|
|||
apply plugin: 'elasticsearch.vagrantsupport'
|
||||
|
||||
versions << [
|
||||
'hadoop2': '2.7.1'
|
||||
'hadoop2': '2.8.1'
|
||||
]
|
||||
|
||||
configurations {
|
||||
|
|
@ -45,7 +45,8 @@ dependencies {
|
|||
compile "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}"
|
||||
compile "org.apache.hadoop:hadoop-auth:${versions.hadoop2}"
|
||||
compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
|
||||
compile 'org.apache.htrace:htrace-core:3.1.0-incubating'
|
||||
compile "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop2}"
|
||||
compile 'org.apache.htrace:htrace-core4:4.0.1-incubating'
|
||||
compile 'com.google.guava:guava:11.0.2'
|
||||
compile 'com.google.protobuf:protobuf-java:2.5.0'
|
||||
compile 'commons-logging:commons-logging:1.1.3'
|
||||
|
|
@ -212,9 +213,12 @@ thirdPartyAudit.excludes = [
|
|||
'com.google.gson.stream.JsonReader',
|
||||
'com.google.gson.stream.JsonWriter',
|
||||
'com.jcraft.jsch.ChannelExec',
|
||||
'com.jcraft.jsch.ChannelSftp',
|
||||
'com.jcraft.jsch.ChannelSftp$LsEntry',
|
||||
'com.jcraft.jsch.JSch',
|
||||
'com.jcraft.jsch.Logger',
|
||||
'com.jcraft.jsch.Session',
|
||||
'com.jcraft.jsch.SftpATTRS',
|
||||
'com.sun.jersey.api.ParamException',
|
||||
'com.sun.jersey.api.core.HttpContext',
|
||||
'com.sun.jersey.core.spi.component.ComponentContext',
|
||||
|
|
@ -239,6 +243,7 @@ thirdPartyAudit.excludes = [
|
|||
'io.netty.channel.ChannelHandlerContext',
|
||||
'io.netty.channel.ChannelInboundHandlerAdapter',
|
||||
'io.netty.channel.ChannelInitializer',
|
||||
'io.netty.channel.ChannelOption',
|
||||
'io.netty.channel.ChannelPipeline',
|
||||
'io.netty.channel.EventLoopGroup',
|
||||
'io.netty.channel.SimpleChannelInboundHandler',
|
||||
|
|
@ -267,6 +272,7 @@ thirdPartyAudit.excludes = [
|
|||
'io.netty.handler.stream.ChunkedStream',
|
||||
'io.netty.handler.stream.ChunkedWriteHandler',
|
||||
'io.netty.util.concurrent.GlobalEventExecutor',
|
||||
'io.netty.util.ReferenceCountUtil',
|
||||
'javax.ws.rs.core.Context',
|
||||
'javax.ws.rs.core.MediaType',
|
||||
'javax.ws.rs.core.MultivaluedMap',
|
||||
|
|
@ -317,7 +323,6 @@ thirdPartyAudit.excludes = [
|
|||
'org.apache.commons.digester.substitution.MultiVariableExpander',
|
||||
'org.apache.commons.digester.substitution.VariableSubstitutor',
|
||||
'org.apache.commons.digester.xmlrules.DigesterLoader',
|
||||
'org.apache.commons.httpclient.util.URIUtil',
|
||||
'org.apache.commons.jxpath.JXPathContext',
|
||||
'org.apache.commons.jxpath.ri.JXPathContextReferenceImpl',
|
||||
'org.apache.commons.jxpath.ri.QName',
|
||||
|
|
@ -369,6 +374,7 @@ thirdPartyAudit.excludes = [
|
|||
'org.apache.curator.utils.PathUtils',
|
||||
'org.apache.curator.utils.ThreadUtils',
|
||||
'org.apache.curator.utils.ZKPaths',
|
||||
'org.apache.directory.shared.kerberos.components.EncryptionKey',
|
||||
'org.apache.directory.server.kerberos.shared.keytab.Keytab',
|
||||
'org.apache.directory.server.kerberos.shared.keytab.KeytabEntry',
|
||||
'org.apache.http.NameValuePair',
|
||||
|
|
@ -402,13 +408,11 @@ thirdPartyAudit.excludes = [
|
|||
'org.codehaus.jackson.JsonFactory',
|
||||
'org.codehaus.jackson.JsonGenerator',
|
||||
'org.codehaus.jackson.JsonGenerator$Feature',
|
||||
'org.codehaus.jackson.JsonNode',
|
||||
'org.codehaus.jackson.map.MappingJsonFactory',
|
||||
'org.codehaus.jackson.map.ObjectMapper',
|
||||
'org.codehaus.jackson.map.ObjectReader',
|
||||
'org.codehaus.jackson.map.ObjectWriter',
|
||||
'org.codehaus.jackson.node.ContainerNode',
|
||||
'org.codehaus.jackson.type.TypeReference',
|
||||
'org.codehaus.jackson.util.MinimalPrettyPrinter',
|
||||
'org.fusesource.leveldbjni.JniDBFactory',
|
||||
'org.iq80.leveldb.DB',
|
||||
|
|
@ -427,6 +431,7 @@ thirdPartyAudit.excludes = [
|
|||
'org.mortbay.jetty.handler.HandlerCollection',
|
||||
'org.mortbay.jetty.handler.RequestLogHandler',
|
||||
'org.mortbay.jetty.nio.SelectChannelConnector',
|
||||
'org.mortbay.jetty.security.SslSelectChannelConnector',
|
||||
'org.mortbay.jetty.security.SslSocketConnector',
|
||||
'org.mortbay.jetty.servlet.AbstractSessionManager',
|
||||
'org.mortbay.jetty.servlet.Context',
|
||||
|
|
@ -437,7 +442,6 @@ thirdPartyAudit.excludes = [
|
|||
'org.mortbay.jetty.servlet.ServletHolder',
|
||||
'org.mortbay.jetty.servlet.SessionHandler',
|
||||
'org.mortbay.jetty.webapp.WebAppContext',
|
||||
'org.mortbay.log.Log',
|
||||
'org.mortbay.thread.QueuedThreadPool',
|
||||
'org.mortbay.util.MultiException',
|
||||
'org.mortbay.util.ajax.JSON$Convertible',
|
||||
|
|
@ -473,9 +477,26 @@ thirdPartyAudit.excludes = [
|
|||
'org.apache.log4j.AppenderSkeleton',
|
||||
'org.apache.log4j.AsyncAppender',
|
||||
'org.apache.log4j.helpers.ISO8601DateFormat',
|
||||
'org.apache.log4j.spi.ThrowableInformation'
|
||||
'org.apache.log4j.spi.ThrowableInformation',
|
||||
|
||||
// New optional dependencies in 2.8
|
||||
'com.nimbusds.jose.JWSObject$State',
|
||||
'com.nimbusds.jose.crypto.RSASSAVerifier',
|
||||
'com.nimbusds.jwt.ReadOnlyJWTClaimsSet',
|
||||
'com.nimbusds.jwt.SignedJWT',
|
||||
'com.squareup.okhttp.Call',
|
||||
'com.squareup.okhttp.MediaType',
|
||||
'com.squareup.okhttp.OkHttpClient',
|
||||
'com.squareup.okhttp.Request$Builder',
|
||||
'com.squareup.okhttp.RequestBody',
|
||||
'com.squareup.okhttp.Response',
|
||||
'com.squareup.okhttp.ResponseBody'
|
||||
]
|
||||
|
||||
if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
|
||||
thirdPartyAudit.excludes.add('javax.xml.bind.annotation.adapters.HexBinaryAdapter')
|
||||
}
|
||||
|
||||
// Gradle 2.13 bundles org.slf4j.impl.StaticLoggerBinder in its core.jar which leaks into the forbidden APIs ant task
|
||||
// Gradle 2.14+ does not bundle this class anymore so we need to properly exclude it here.
|
||||
if (GradleVersion.current() > GradleVersion.version("2.13")) {
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
2a77fe74ee056bf45598cf7e20cd624e8388e627
|
||||
|
|
@ -0,0 +1 @@
|
|||
335a867cf42bf789919bfc3229ff26747124e8f1
|
||||
|
|
@ -1 +0,0 @@
|
|||
2515f339f97f1d7ba850485e06e395a58586bc2e
|
||||
|
|
@ -0,0 +1 @@
|
|||
688ccccc0e0739d8737a93b0039a4a661e52084b
|
||||
|
|
@ -1 +0,0 @@
|
|||
dbc2faacd210e6a1e3eb7def6e42065c7457d960
|
||||
|
|
@ -0,0 +1 @@
|
|||
4812f251f8100fd4722c3cec5d7353f71f69cda9
|
||||
|
|
@ -1 +0,0 @@
|
|||
50580f5ebab60b1b318ad157f668d8e40a1cc0da
|
||||
|
|
@ -0,0 +1 @@
|
|||
a4df18b79e4d0349ce4b58a52d314e7ae1d6be99
|
||||
|
|
@ -1 +0,0 @@
|
|||
11681de93a4cd76c841e352b7094f839b072a21f
|
||||
|
|
@ -0,0 +1 @@
|
|||
a378f4bc8e6cd779d779c9f512e0e31edd771633
|
||||
|
|
@ -0,0 +1 @@
|
|||
6b0100e4f58ecf7ce75817fce1ffdfbec947337a
|
||||
|
|
@ -1 +0,0 @@
|
|||
f73606e7c9ede5802335c290bf47490ad6d51df3
|
||||
|
|
@ -0,0 +1 @@
|
|||
f4ef727cb4675788ac66f48e217020acc1690960
|
||||
|
|
@ -203,8 +203,8 @@ of dependencies that are NOT Apache Licensed.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
The HTrace Owl logo is from http://www.clker.com/clipart-13653.html. It is
|
||||
public domain.
|
||||
Units, a string formatting go library, is Copyright (c) 2014 Alec Thomas
|
||||
and MIT licensed: https://github.com/alecthomas/units/blob/master/COPYING
|
||||
|
||||
D3, a javascript library for manipulating data, used by htrace-hbase
|
||||
is Copyright 2010-2014, Michael Bostock and BSD licensed:
|
||||
|
|
@ -239,4 +239,7 @@ https://github.com/moment/moment/blob/develop/LICENSE
|
|||
CMP is an implementation of the MessagePack serialization format in
|
||||
C. It is licensed under the MIT license:
|
||||
https://github.com/camgunz/cmp/blob/master/LICENSE
|
||||
See ./htrace-c/src/util/cmp.c and ./htrace-c/src/util/cmp.h.
|
||||
|
||||
go-codec is an implementation of several serialization and deserialization
|
||||
codecs in Go. It is licensed under the MIT license:
|
||||
https://github.com/ugorji/go/blob/master/LICENSE
|
||||
|
|
@ -11,3 +11,6 @@ that are NOT Apache licensed (with pointers to their licensing)
|
|||
Apache HTrace includes an Apache Thrift connector to Zipkin. Zipkin
|
||||
is a distributed tracing system that is Apache 2.0 Licensed.
|
||||
Copyright 2012 Twitter, Inc.
|
||||
|
||||
Our Owl logo we took from http://www.clker.com/clipart-13653.html.
|
||||
It is public domain/free.
|
||||
|
|
@ -62,8 +62,6 @@ public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
|
|||
Class.forName("org.apache.hadoop.util.StringUtils");
|
||||
Class.forName("org.apache.hadoop.util.ShutdownHookManager");
|
||||
Class.forName("org.apache.hadoop.conf.Configuration");
|
||||
Class.forName("org.apache.hadoop.hdfs.protocol.HdfsConstants");
|
||||
Class.forName("org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck");
|
||||
} catch (ClassNotFoundException | IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -19,15 +19,6 @@
|
|||
|
||||
package org.elasticsearch.repositories.hdfs;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.AbstractFileSystem;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
||||
import org.elasticsearch.common.SuppressForbidden;
|
||||
import org.elasticsearch.common.blobstore.BlobStore;
|
||||
import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
|
||||
|
||||
import javax.security.auth.Subject;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
|
|
@ -38,7 +29,18 @@ import java.security.PrivilegedAction;
|
|||
import java.security.PrivilegedActionException;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.util.Collections;
|
||||
import javax.security.auth.Subject;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.AbstractFileSystem;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
||||
import org.elasticsearch.common.SuppressForbidden;
|
||||
import org.elasticsearch.common.blobstore.BlobStore;
|
||||
import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
|
||||
|
||||
@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
|
||||
public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.repositories.hdfs;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.ThreadFilter;
|
||||
|
||||
/**
|
||||
* In Hadoop 2.8.0, there is a thread that is started by the filesystem to clean up old execution stats.
|
||||
* This thread ignores all interrupts, catching InterruptedException, logging it, and continuing on
|
||||
* with its work. The thread is a daemon, so it thankfully does not stop the JVM from closing, and it
|
||||
* is started only once in a class's static initialization. This currently breaks our testing as this
|
||||
* thread leaks out of the client and is picked up by the test framework. This thread filter is meant
|
||||
* to ignore the offending thread until a version of Hadoop is released that addresses the incorrect
|
||||
* interrupt handling.
|
||||
*
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HADOOP-12829">https://issues.apache.org/jira/browse/HADOOP-12829</a>
|
||||
* @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner"
|
||||
* @see "org.apache.hadoop.fs.FileSystem.Statistics"
|
||||
*/
|
||||
public final class HdfsClientThreadLeakFilter implements ThreadFilter {
|
||||
|
||||
private static final String OFFENDING_THREAD_NAME =
|
||||
"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner";
|
||||
|
||||
@Override
|
||||
public boolean reject(Thread t) {
|
||||
return t.getName().equals(OFFENDING_THREAD_NAME);
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.greaterThan;
|
|||
|
||||
import java.util.Collection;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
|
||||
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
|
||||
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
|
||||
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
|
||||
|
|
@ -34,6 +35,7 @@ import org.elasticsearch.repositories.RepositoryException;
|
|||
import org.elasticsearch.snapshots.SnapshotState;
|
||||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
|
||||
@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
|
||||
public class HdfsTests extends ESSingleNodeTestCase {
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
apply plugin: 'elasticsearch.build'
|
||||
|
||||
versions << [
|
||||
'hadoop2': '2.7.1'
|
||||
'hadoop2': '2.8.1'
|
||||
]
|
||||
|
||||
// we create MiniHdfsCluster with the hadoop artifact
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import java.nio.file.Path;
|
|||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
|
@ -49,7 +50,8 @@ public class MiniHDFS {
|
|||
|
||||
public static void main(String[] args) throws Exception {
|
||||
if (args.length != 1 && args.length != 3) {
|
||||
throw new IllegalArgumentException("MiniHDFS <baseDirectory> [<kerberosPrincipal> <kerberosKeytab>]");
|
||||
throw new IllegalArgumentException("Expected: MiniHDFS <baseDirectory> [<kerberosPrincipal> <kerberosKeytab>], " +
|
||||
"got: " + Arrays.toString(args));
|
||||
}
|
||||
boolean secure = args.length == 3;
|
||||
|
||||
|
|
@ -83,6 +85,7 @@ public class MiniHDFS {
|
|||
cfg.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, keytabFile);
|
||||
cfg.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, keytabFile);
|
||||
cfg.set(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, "true");
|
||||
cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true");
|
||||
cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true");
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue