mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-06 13:08:29 +00:00
b9622251fe
This commit sets the version on the repository-hdfs Guava dependency to version 11.0.2. This change is made to align the version here with the version that is defined in the POM for Hadoop 2.7.1, the version of Hadoop that the repository-hdfs plugin is based on. See HADOOP-10101 and HADOOP-11319 for the ridiculous history of trying to upgrade Guava past this version in the Hadoop project. Relates #23420
372 lines
16 KiB
Groovy
372 lines
16 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import java.nio.file.Files
|
|
import java.nio.file.Path
|
|
import java.nio.file.Paths
|
|
|
|
esplugin {
|
|
description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
|
|
classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'hadoop2': '2.7.1'
|
|
]
|
|
|
|
configurations {
|
|
hdfsFixture
|
|
}
|
|
|
|
dependencies {
|
|
compile "org.apache.hadoop:hadoop-client:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-common:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-auth:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
|
|
compile 'org.apache.htrace:htrace-core:3.1.0-incubating'
|
|
compile 'com.google.guava:guava:11.0.2'
|
|
compile 'com.google.protobuf:protobuf-java:2.5.0'
|
|
compile 'commons-logging:commons-logging:1.1.3'
|
|
compile 'commons-cli:commons-cli:1.2'
|
|
compile 'commons-collections:commons-collections:3.2.2'
|
|
compile 'commons-configuration:commons-configuration:1.6'
|
|
compile 'commons-io:commons-io:2.4'
|
|
compile 'commons-lang:commons-lang:2.6'
|
|
compile 'javax.servlet:servlet-api:2.5'
|
|
compile "org.slf4j:slf4j-api:${versions.slf4j}"
|
|
|
|
hdfsFixture project(':test:fixtures:hdfs-fixture')
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /hadoop-.*/, to: 'hadoop'
|
|
}
|
|
|
|
task hdfsFixture(type: org.elasticsearch.gradle.test.Fixture) {
|
|
dependsOn project.configurations.hdfsFixture
|
|
executable = new File(project.javaHome, 'bin/java')
|
|
env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
|
|
args 'hdfs.MiniHDFS',
|
|
baseDir
|
|
}
|
|
|
|
boolean fixtureSupported = false;
|
|
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
|
// hdfs fixture will not start without hadoop native libraries on windows
|
|
String nativePath = System.getenv("HADOOP_HOME")
|
|
if (nativePath != null) {
|
|
Path path = Paths.get(nativePath);
|
|
if (Files.isDirectory(path) &&
|
|
Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
|
|
Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
|
|
Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
|
|
fixtureSupported = true
|
|
} else {
|
|
throw new IllegalStateException("HADOOP_HOME: " + path.toString() + " is invalid, does not contain hadoop native libraries in $HADOOP_HOME/bin");
|
|
}
|
|
}
|
|
} else {
|
|
fixtureSupported = true
|
|
}
|
|
|
|
if (fixtureSupported) {
|
|
integTest.dependsOn hdfsFixture
|
|
} else {
|
|
logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
|
|
// just tests that the plugin loads
|
|
integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
|
|
}
|
|
|
|
thirdPartyAudit.excludes = [
|
|
// classes are missing, because we added hadoop jars one by one until tests pass.
|
|
'com.google.gson.stream.JsonReader',
|
|
'com.google.gson.stream.JsonWriter',
|
|
'com.jcraft.jsch.ChannelExec',
|
|
'com.jcraft.jsch.JSch',
|
|
'com.jcraft.jsch.Logger',
|
|
'com.jcraft.jsch.Session',
|
|
'com.sun.jersey.api.ParamException',
|
|
'com.sun.jersey.api.core.HttpContext',
|
|
'com.sun.jersey.core.spi.component.ComponentContext',
|
|
'com.sun.jersey.core.spi.component.ComponentScope',
|
|
'com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable',
|
|
'com.sun.jersey.spi.container.ContainerRequest',
|
|
'com.sun.jersey.spi.container.ContainerRequestFilter',
|
|
'com.sun.jersey.spi.container.ContainerResponseFilter',
|
|
'com.sun.jersey.spi.container.ResourceFilter',
|
|
'com.sun.jersey.spi.container.servlet.ServletContainer',
|
|
'com.sun.jersey.spi.inject.Injectable',
|
|
'com.sun.jersey.spi.inject.InjectableProvider',
|
|
'io.netty.bootstrap.Bootstrap',
|
|
'io.netty.bootstrap.ChannelFactory',
|
|
'io.netty.bootstrap.ServerBootstrap',
|
|
'io.netty.buffer.ByteBuf',
|
|
'io.netty.buffer.Unpooled',
|
|
'io.netty.channel.Channel',
|
|
'io.netty.channel.ChannelFuture',
|
|
'io.netty.channel.ChannelFutureListener',
|
|
'io.netty.channel.ChannelHandler',
|
|
'io.netty.channel.ChannelHandlerContext',
|
|
'io.netty.channel.ChannelInboundHandlerAdapter',
|
|
'io.netty.channel.ChannelInitializer',
|
|
'io.netty.channel.ChannelPipeline',
|
|
'io.netty.channel.EventLoopGroup',
|
|
'io.netty.channel.SimpleChannelInboundHandler',
|
|
'io.netty.channel.group.ChannelGroup',
|
|
'io.netty.channel.group.ChannelGroupFuture',
|
|
'io.netty.channel.group.DefaultChannelGroup',
|
|
'io.netty.channel.nio.NioEventLoopGroup',
|
|
'io.netty.channel.socket.SocketChannel',
|
|
'io.netty.channel.socket.nio.NioServerSocketChannel',
|
|
'io.netty.channel.socket.nio.NioSocketChannel',
|
|
'io.netty.handler.codec.http.DefaultFullHttpRequest',
|
|
'io.netty.handler.codec.http.DefaultFullHttpResponse',
|
|
'io.netty.handler.codec.http.DefaultHttpResponse',
|
|
'io.netty.handler.codec.http.HttpContent',
|
|
'io.netty.handler.codec.http.HttpHeaders',
|
|
'io.netty.handler.codec.http.HttpMethod',
|
|
'io.netty.handler.codec.http.HttpRequest',
|
|
'io.netty.handler.codec.http.HttpRequestDecoder',
|
|
'io.netty.handler.codec.http.HttpRequestEncoder',
|
|
'io.netty.handler.codec.http.HttpResponseEncoder',
|
|
'io.netty.handler.codec.http.HttpResponseStatus',
|
|
'io.netty.handler.codec.http.HttpVersion',
|
|
'io.netty.handler.codec.http.QueryStringDecoder',
|
|
'io.netty.handler.codec.string.StringEncoder',
|
|
'io.netty.handler.ssl.SslHandler',
|
|
'io.netty.handler.stream.ChunkedStream',
|
|
'io.netty.handler.stream.ChunkedWriteHandler',
|
|
'io.netty.util.concurrent.GlobalEventExecutor',
|
|
'javax.ws.rs.core.Context',
|
|
'javax.ws.rs.core.MediaType',
|
|
'javax.ws.rs.core.MultivaluedMap',
|
|
'javax.ws.rs.core.Response$ResponseBuilder',
|
|
'javax.ws.rs.core.Response$Status',
|
|
'javax.ws.rs.core.Response',
|
|
'javax.ws.rs.core.StreamingOutput',
|
|
'javax.ws.rs.core.UriBuilder',
|
|
'javax.ws.rs.ext.ExceptionMapper',
|
|
'jdiff.JDiff',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.avro.Schema',
|
|
'org.apache.avro.file.DataFileReader',
|
|
'org.apache.avro.file.FileReader',
|
|
'org.apache.avro.file.SeekableInput',
|
|
'org.apache.avro.generic.GenericDatumReader',
|
|
'org.apache.avro.generic.GenericDatumWriter',
|
|
'org.apache.avro.io.BinaryDecoder',
|
|
'org.apache.avro.io.BinaryEncoder',
|
|
'org.apache.avro.io.DatumReader',
|
|
'org.apache.avro.io.DatumWriter',
|
|
'org.apache.avro.io.DecoderFactory',
|
|
'org.apache.avro.io.EncoderFactory',
|
|
'org.apache.avro.io.JsonEncoder',
|
|
'org.apache.avro.reflect.ReflectData',
|
|
'org.apache.avro.reflect.ReflectDatumReader',
|
|
'org.apache.avro.reflect.ReflectDatumWriter',
|
|
'org.apache.avro.specific.SpecificDatumReader',
|
|
'org.apache.avro.specific.SpecificDatumWriter',
|
|
'org.apache.avro.specific.SpecificRecord',
|
|
'org.apache.commons.beanutils.BeanUtils',
|
|
'org.apache.commons.beanutils.DynaBean',
|
|
'org.apache.commons.beanutils.DynaClass',
|
|
'org.apache.commons.beanutils.DynaProperty',
|
|
'org.apache.commons.beanutils.PropertyUtils',
|
|
'org.apache.commons.compress.archivers.tar.TarArchiveEntry',
|
|
'org.apache.commons.compress.archivers.tar.TarArchiveInputStream',
|
|
'org.apache.commons.codec.DecoderException',
|
|
'org.apache.commons.codec.binary.Base64',
|
|
'org.apache.commons.codec.binary.Hex',
|
|
'org.apache.commons.codec.digest.DigestUtils',
|
|
'org.apache.commons.daemon.Daemon',
|
|
'org.apache.commons.daemon.DaemonContext',
|
|
'org.apache.commons.digester.AbstractObjectCreationFactory',
|
|
'org.apache.commons.digester.CallMethodRule',
|
|
'org.apache.commons.digester.Digester',
|
|
'org.apache.commons.digester.ObjectCreationFactory',
|
|
'org.apache.commons.digester.substitution.MultiVariableExpander',
|
|
'org.apache.commons.digester.substitution.VariableSubstitutor',
|
|
'org.apache.commons.digester.xmlrules.DigesterLoader',
|
|
'org.apache.commons.httpclient.util.URIUtil',
|
|
'org.apache.commons.jxpath.JXPathContext',
|
|
'org.apache.commons.jxpath.ri.JXPathContextReferenceImpl',
|
|
'org.apache.commons.jxpath.ri.QName',
|
|
'org.apache.commons.jxpath.ri.compiler.NodeNameTest',
|
|
'org.apache.commons.jxpath.ri.compiler.NodeTest',
|
|
'org.apache.commons.jxpath.ri.compiler.NodeTypeTest',
|
|
'org.apache.commons.jxpath.ri.model.NodeIterator',
|
|
'org.apache.commons.jxpath.ri.model.NodePointer',
|
|
'org.apache.commons.jxpath.ri.model.NodePointerFactory',
|
|
'org.apache.commons.math3.util.ArithmeticUtils',
|
|
'org.apache.commons.net.ftp.FTPClient',
|
|
'org.apache.commons.net.ftp.FTPFile',
|
|
'org.apache.commons.net.ftp.FTPReply',
|
|
'org.apache.commons.net.util.SubnetUtils$SubnetInfo',
|
|
'org.apache.commons.net.util.SubnetUtils',
|
|
'org.apache.curator.ensemble.fixed.FixedEnsembleProvider',
|
|
'org.apache.curator.framework.CuratorFramework',
|
|
'org.apache.curator.framework.CuratorFrameworkFactory$Builder',
|
|
'org.apache.curator.framework.CuratorFrameworkFactory',
|
|
'org.apache.curator.framework.api.ACLBackgroundPathAndBytesable',
|
|
'org.apache.curator.framework.api.ACLProvider',
|
|
'org.apache.curator.framework.api.BackgroundPathAndBytesable',
|
|
'org.apache.curator.framework.api.ChildrenDeletable',
|
|
'org.apache.curator.framework.api.CreateBuilder',
|
|
'org.apache.curator.framework.api.DeleteBuilder',
|
|
'org.apache.curator.framework.api.ExistsBuilder',
|
|
'org.apache.curator.framework.api.GetChildrenBuilder',
|
|
'org.apache.curator.framework.api.GetDataBuilder',
|
|
'org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable',
|
|
'org.apache.curator.framework.api.SetDataBuilder',
|
|
'org.apache.curator.framework.api.WatchPathable',
|
|
'org.apache.curator.framework.imps.DefaultACLProvider',
|
|
'org.apache.curator.framework.listen.ListenerContainer',
|
|
'org.apache.curator.framework.recipes.cache.ChildData',
|
|
'org.apache.curator.framework.recipes.cache.PathChildrenCache$StartMode',
|
|
'org.apache.curator.framework.recipes.cache.PathChildrenCache',
|
|
'org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent$Type',
|
|
'org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent',
|
|
'org.apache.curator.framework.recipes.cache.PathChildrenCacheListener',
|
|
'org.apache.curator.framework.recipes.locks.Reaper$Mode',
|
|
'org.apache.curator.framework.recipes.locks.Reaper',
|
|
'org.apache.curator.framework.recipes.shared.SharedCount',
|
|
'org.apache.curator.framework.recipes.shared.VersionedValue',
|
|
'org.apache.curator.retry.ExponentialBackoffRetry',
|
|
'org.apache.curator.retry.RetryNTimes',
|
|
'org.apache.curator.utils.CloseableScheduledExecutorService',
|
|
'org.apache.curator.utils.CloseableUtils',
|
|
'org.apache.curator.utils.EnsurePath',
|
|
'org.apache.curator.utils.PathUtils',
|
|
'org.apache.curator.utils.ThreadUtils',
|
|
'org.apache.curator.utils.ZKPaths',
|
|
'org.apache.directory.server.kerberos.shared.keytab.Keytab',
|
|
'org.apache.directory.server.kerberos.shared.keytab.KeytabEntry',
|
|
'org.apache.http.NameValuePair',
|
|
'org.apache.http.client.utils.URIBuilder',
|
|
'org.apache.http.client.utils.URLEncodedUtils',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'org.apache.tools.ant.BuildException',
|
|
'org.apache.tools.ant.DirectoryScanner',
|
|
'org.apache.tools.ant.Task',
|
|
'org.apache.tools.ant.taskdefs.Execute',
|
|
'org.apache.tools.ant.types.FileSet',
|
|
'org.apache.xml.serialize.OutputFormat',
|
|
'org.apache.xml.serialize.XMLSerializer',
|
|
'org.apache.zookeeper.AsyncCallback$StatCallback',
|
|
'org.apache.zookeeper.AsyncCallback$StringCallback',
|
|
'org.apache.zookeeper.CreateMode',
|
|
'org.apache.zookeeper.KeeperException$Code',
|
|
'org.apache.zookeeper.KeeperException',
|
|
'org.apache.zookeeper.WatchedEvent',
|
|
'org.apache.zookeeper.Watcher$Event$EventType',
|
|
'org.apache.zookeeper.Watcher$Event$KeeperState',
|
|
'org.apache.zookeeper.Watcher',
|
|
'org.apache.zookeeper.ZKUtil',
|
|
'org.apache.zookeeper.ZooDefs$Ids',
|
|
'org.apache.zookeeper.ZooKeeper',
|
|
'org.apache.zookeeper.data.ACL',
|
|
'org.apache.zookeeper.data.Id',
|
|
'org.apache.zookeeper.data.Stat',
|
|
'org.codehaus.jackson.JsonEncoding',
|
|
'org.codehaus.jackson.JsonFactory',
|
|
'org.codehaus.jackson.JsonGenerator',
|
|
'org.codehaus.jackson.JsonGenerator$Feature',
|
|
'org.codehaus.jackson.JsonNode',
|
|
'org.codehaus.jackson.map.MappingJsonFactory',
|
|
'org.codehaus.jackson.map.ObjectMapper',
|
|
'org.codehaus.jackson.map.ObjectReader',
|
|
'org.codehaus.jackson.map.ObjectWriter',
|
|
'org.codehaus.jackson.node.ContainerNode',
|
|
'org.codehaus.jackson.type.TypeReference',
|
|
'org.codehaus.jackson.util.MinimalPrettyPrinter',
|
|
'org.fusesource.leveldbjni.JniDBFactory',
|
|
'org.iq80.leveldb.DB',
|
|
'org.iq80.leveldb.Options',
|
|
'org.iq80.leveldb.WriteBatch',
|
|
'org.mortbay.jetty.Connector',
|
|
'org.mortbay.jetty.Handler',
|
|
'org.mortbay.jetty.InclusiveByteRange',
|
|
'org.mortbay.jetty.MimeTypes',
|
|
'org.mortbay.jetty.NCSARequestLog',
|
|
'org.mortbay.jetty.RequestLog',
|
|
'org.mortbay.jetty.Server',
|
|
'org.mortbay.jetty.handler.ContextHandler$SContext',
|
|
'org.mortbay.jetty.handler.ContextHandler',
|
|
'org.mortbay.jetty.handler.ContextHandlerCollection',
|
|
'org.mortbay.jetty.handler.HandlerCollection',
|
|
'org.mortbay.jetty.handler.RequestLogHandler',
|
|
'org.mortbay.jetty.nio.SelectChannelConnector',
|
|
'org.mortbay.jetty.security.SslSocketConnector',
|
|
'org.mortbay.jetty.servlet.AbstractSessionManager',
|
|
'org.mortbay.jetty.servlet.Context',
|
|
'org.mortbay.jetty.servlet.DefaultServlet',
|
|
'org.mortbay.jetty.servlet.FilterHolder',
|
|
'org.mortbay.jetty.servlet.FilterMapping',
|
|
'org.mortbay.jetty.servlet.ServletHandler',
|
|
'org.mortbay.jetty.servlet.ServletHolder',
|
|
'org.mortbay.jetty.servlet.SessionHandler',
|
|
'org.mortbay.jetty.webapp.WebAppContext',
|
|
'org.mortbay.log.Log',
|
|
'org.mortbay.thread.QueuedThreadPool',
|
|
'org.mortbay.util.MultiException',
|
|
'org.mortbay.util.ajax.JSON$Convertible',
|
|
'org.mortbay.util.ajax.JSON$Output',
|
|
'org.mortbay.util.ajax.JSON',
|
|
'org.znerd.xmlenc.XMLOutputter',
|
|
|
|
// internal java api: sun.net.dns.ResolverConfiguration
|
|
// internal java api: sun.net.util.IPAddressUtil
|
|
'org.apache.hadoop.security.SecurityUtil$QualifiedHostResolver',
|
|
|
|
// internal java api: sun.misc.Unsafe
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
|
|
'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1',
|
|
'org.apache.hadoop.io.nativeio.NativeIO',
|
|
'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm',
|
|
'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot',
|
|
|
|
// internal java api: sun.nio.ch.DirectBuffer
|
|
// internal java api: sun.misc.Cleaner
|
|
'org.apache.hadoop.io.nativeio.NativeIO$POSIX',
|
|
'org.apache.hadoop.crypto.CryptoStreamUtils',
|
|
|
|
// internal java api: sun.misc.SignalHandler
|
|
'org.apache.hadoop.util.SignalLogger$Handler',
|
|
|
|
// optional dependencies of slf4j-api
|
|
'org.slf4j.impl.StaticMDCBinder',
|
|
'org.slf4j.impl.StaticMarkerBinder',
|
|
|
|
'org.apache.log4j.AppenderSkeleton',
|
|
'org.apache.log4j.AsyncAppender',
|
|
'org.apache.log4j.helpers.ISO8601DateFormat',
|
|
'org.apache.log4j.spi.ThrowableInformation'
|
|
]
|
|
|
|
// Gradle 2.13 bundles org.slf4j.impl.StaticLoggerBinder in its core.jar which leaks into the forbidden APIs ant task
|
|
// Gradle 2.14+ does not bundle this class anymore so we need to properly exclude it here.
|
|
if (GradleVersion.current() > GradleVersion.version("2.13")) {
|
|
thirdPartyAudit.excludes += ['org.slf4j.impl.StaticLoggerBinder']
|
|
}
|