OpenSearch/plugins/repository-hdfs/build.gradle

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.tools.ant.taskdefs.condition.Os
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
 
esplugin {
  description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
  classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
}

versions << [
  'hadoop2': '2.7.1'
]

configurations {
  hdfsFixture
}

dependencies {
  compile "org.apache.hadoop:hadoop-client:${versions.hadoop2}"
  compile "org.apache.hadoop:hadoop-common:${versions.hadoop2}"
  compile "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}"
  compile "org.apache.hadoop:hadoop-auth:${versions.hadoop2}"
  compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
  compile 'org.apache.htrace:htrace-core:3.1.0-incubating'
  compile 'com.google.guava:guava:16.0.1'
  compile 'com.google.protobuf:protobuf-java:2.5.0'
  compile 'commons-logging:commons-logging:1.1.3'
  compile 'commons-collections:commons-collections:3.2.2'
  compile 'commons-configuration:commons-configuration:1.6'
  compile 'commons-io:commons-io:2.4'
  compile 'commons-lang:commons-lang:2.6'
  compile 'javax.servlet:servlet-api:2.5'
  compile "org.slf4j:slf4j-api:${versions.slf4j}"

  hdfsFixture project(':test:fixtures:hdfs-fixture')
}

dependencyLicenses {
  mapping from: /hadoop-.*/, to: 'hadoop'
}

task hdfsFixture(type: org.elasticsearch.gradle.test.Fixture) {
  dependsOn project.configurations.hdfsFixture
  executable = new File(project.javaHome, 'bin/java')
  env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
  args 'hdfs.MiniHDFS',
       baseDir
}

integTest {
  boolean fixtureSupported = false;
  if (Os.isFamily(Os.FAMILY_WINDOWS)) {
    // hdfs fixture will not start without hadoop native libraries on windows
    String nativePath = System.getenv("HADOOP_HOME")
    if (nativePath != null) {
      Path path = Paths.get(nativePath);
      if (Files.isDirectory(path) &&
          Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
          Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
          Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
        fixtureSupported = true
      } else {
        throw new IllegalStateException("HADOOP_HOME: " + path.toString() + " is invalid, does not contain hadoop native libraries in $HADOOP_HOME/bin");
      }
    }
  } else {
    fixtureSupported = true
  }

  if (fixtureSupported) {
    dependsOn hdfsFixture
  } else {
    logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
    // just tests that the plugin loads
    systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
  }
}

thirdPartyAudit.excludes = [
  // classes are missing, because we added hadoop jars one by one until tests pass.
  'com.google.gson.stream.JsonReader', 
  'com.google.gson.stream.JsonWriter', 
  'com.jcraft.jsch.ChannelExec', 
  'com.jcraft.jsch.JSch', 
  'com.jcraft.jsch.Logger', 
  'com.jcraft.jsch.Session', 
  'com.sun.jersey.api.ParamException', 
  'com.sun.jersey.api.core.HttpContext', 
  'com.sun.jersey.core.spi.component.ComponentContext', 
  'com.sun.jersey.core.spi.component.ComponentScope', 
  'com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable', 
  'com.sun.jersey.spi.container.ContainerRequest', 
  'com.sun.jersey.spi.container.ContainerRequestFilter', 
  'com.sun.jersey.spi.container.ContainerResponseFilter', 
  'com.sun.jersey.spi.container.ResourceFilter', 
  'com.sun.jersey.spi.container.servlet.ServletContainer', 
  'com.sun.jersey.spi.inject.Injectable', 
  'com.sun.jersey.spi.inject.InjectableProvider',
  'io.netty.bootstrap.Bootstrap', 
  'io.netty.bootstrap.ChannelFactory', 
  'io.netty.bootstrap.ServerBootstrap', 
  'io.netty.buffer.ByteBuf', 
  'io.netty.buffer.Unpooled', 
  'io.netty.channel.Channel', 
  'io.netty.channel.ChannelFuture', 
  'io.netty.channel.ChannelFutureListener', 
  'io.netty.channel.ChannelHandler', 
  'io.netty.channel.ChannelHandlerContext', 
  'io.netty.channel.ChannelInboundHandlerAdapter', 
  'io.netty.channel.ChannelInitializer', 
  'io.netty.channel.ChannelPipeline', 
  'io.netty.channel.EventLoopGroup', 
  'io.netty.channel.SimpleChannelInboundHandler', 
  'io.netty.channel.group.ChannelGroup', 
  'io.netty.channel.group.ChannelGroupFuture', 
  'io.netty.channel.group.DefaultChannelGroup', 
  'io.netty.channel.nio.NioEventLoopGroup', 
  'io.netty.channel.socket.SocketChannel', 
  'io.netty.channel.socket.nio.NioServerSocketChannel', 
  'io.netty.channel.socket.nio.NioSocketChannel', 
  'io.netty.handler.codec.http.DefaultFullHttpRequest', 
  'io.netty.handler.codec.http.DefaultFullHttpResponse', 
  'io.netty.handler.codec.http.DefaultHttpResponse', 
  'io.netty.handler.codec.http.HttpContent', 
  'io.netty.handler.codec.http.HttpHeaders', 
  'io.netty.handler.codec.http.HttpMethod', 
  'io.netty.handler.codec.http.HttpRequest', 
  'io.netty.handler.codec.http.HttpRequestDecoder', 
  'io.netty.handler.codec.http.HttpRequestEncoder', 
  'io.netty.handler.codec.http.HttpResponseEncoder', 
  'io.netty.handler.codec.http.HttpResponseStatus', 
  'io.netty.handler.codec.http.HttpVersion', 
  'io.netty.handler.codec.http.QueryStringDecoder', 
  'io.netty.handler.codec.string.StringEncoder', 
  'io.netty.handler.ssl.SslHandler', 
  'io.netty.handler.stream.ChunkedStream', 
  'io.netty.handler.stream.ChunkedWriteHandler', 
  'io.netty.util.concurrent.GlobalEventExecutor', 
  'javax.ws.rs.core.Context', 
  'javax.ws.rs.core.MediaType', 
  'javax.ws.rs.core.MultivaluedMap', 
  'javax.ws.rs.core.Response$ResponseBuilder', 
  'javax.ws.rs.core.Response$Status', 
  'javax.ws.rs.core.Response', 
  'javax.ws.rs.core.StreamingOutput', 
  'javax.ws.rs.core.UriBuilder', 
  'javax.ws.rs.ext.ExceptionMapper', 
  'jdiff.JDiff', 
  'org.apache.avalon.framework.logger.Logger', 
  'org.apache.avro.Schema', 
  'org.apache.avro.file.DataFileReader', 
  'org.apache.avro.file.FileReader', 
  'org.apache.avro.file.SeekableInput', 
  'org.apache.avro.generic.GenericDatumReader', 
  'org.apache.avro.generic.GenericDatumWriter', 
  'org.apache.avro.io.BinaryDecoder', 
  'org.apache.avro.io.BinaryEncoder', 
  'org.apache.avro.io.DatumReader', 
  'org.apache.avro.io.DatumWriter', 
  'org.apache.avro.io.DecoderFactory', 
  'org.apache.avro.io.EncoderFactory', 
  'org.apache.avro.io.JsonEncoder', 
  'org.apache.avro.reflect.ReflectData', 
  'org.apache.avro.reflect.ReflectDatumReader', 
  'org.apache.avro.reflect.ReflectDatumWriter', 
  'org.apache.avro.specific.SpecificDatumReader', 
  'org.apache.avro.specific.SpecificDatumWriter', 
  'org.apache.avro.specific.SpecificRecord', 
  'org.apache.commons.beanutils.BeanUtils', 
  'org.apache.commons.beanutils.DynaBean', 
  'org.apache.commons.beanutils.DynaClass', 
  'org.apache.commons.beanutils.DynaProperty', 
  'org.apache.commons.beanutils.PropertyUtils', 
  'org.apache.commons.compress.archivers.tar.TarArchiveEntry', 
  'org.apache.commons.compress.archivers.tar.TarArchiveInputStream', 
  'org.apache.commons.codec.DecoderException', 
  'org.apache.commons.codec.binary.Base64', 
  'org.apache.commons.codec.binary.Hex', 
  'org.apache.commons.codec.digest.DigestUtils', 
  'org.apache.commons.daemon.Daemon', 
  'org.apache.commons.daemon.DaemonContext', 
  'org.apache.commons.digester.AbstractObjectCreationFactory', 
  'org.apache.commons.digester.CallMethodRule', 
  'org.apache.commons.digester.Digester', 
  'org.apache.commons.digester.ObjectCreationFactory', 
  'org.apache.commons.digester.substitution.MultiVariableExpander', 
  'org.apache.commons.digester.substitution.VariableSubstitutor', 
  'org.apache.commons.digester.xmlrules.DigesterLoader', 
  'org.apache.commons.httpclient.util.URIUtil', 
  'org.apache.commons.jxpath.JXPathContext', 
  'org.apache.commons.jxpath.ri.JXPathContextReferenceImpl', 
  'org.apache.commons.jxpath.ri.QName', 
  'org.apache.commons.jxpath.ri.compiler.NodeNameTest', 
  'org.apache.commons.jxpath.ri.compiler.NodeTest', 
  'org.apache.commons.jxpath.ri.compiler.NodeTypeTest', 
  'org.apache.commons.jxpath.ri.model.NodeIterator', 
  'org.apache.commons.jxpath.ri.model.NodePointer', 
  'org.apache.commons.jxpath.ri.model.NodePointerFactory', 
  'org.apache.commons.math3.util.ArithmeticUtils', 
  'org.apache.commons.net.ftp.FTPClient', 
  'org.apache.commons.net.ftp.FTPFile', 
  'org.apache.commons.net.ftp.FTPReply', 
  'org.apache.commons.net.util.SubnetUtils$SubnetInfo', 
  'org.apache.commons.net.util.SubnetUtils', 
  'org.apache.curator.ensemble.fixed.FixedEnsembleProvider', 
  'org.apache.curator.framework.CuratorFramework', 
  'org.apache.curator.framework.CuratorFrameworkFactory$Builder', 
  'org.apache.curator.framework.CuratorFrameworkFactory', 
  'org.apache.curator.framework.api.ACLBackgroundPathAndBytesable', 
  'org.apache.curator.framework.api.ACLProvider', 
  'org.apache.curator.framework.api.BackgroundPathAndBytesable', 
  'org.apache.curator.framework.api.ChildrenDeletable', 
  'org.apache.curator.framework.api.CreateBuilder', 
  'org.apache.curator.framework.api.DeleteBuilder', 
  'org.apache.curator.framework.api.ExistsBuilder', 
  'org.apache.curator.framework.api.GetChildrenBuilder', 
  'org.apache.curator.framework.api.GetDataBuilder', 
  'org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable', 
  'org.apache.curator.framework.api.SetDataBuilder', 
  'org.apache.curator.framework.api.WatchPathable', 
  'org.apache.curator.framework.imps.DefaultACLProvider', 
  'org.apache.curator.framework.listen.ListenerContainer', 
  'org.apache.curator.framework.recipes.cache.ChildData', 
  'org.apache.curator.framework.recipes.cache.PathChildrenCache$StartMode', 
  'org.apache.curator.framework.recipes.cache.PathChildrenCache', 
  'org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent$Type', 
  'org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent', 
  'org.apache.curator.framework.recipes.cache.PathChildrenCacheListener', 
  'org.apache.curator.framework.recipes.locks.Reaper$Mode', 
  'org.apache.curator.framework.recipes.locks.Reaper', 
  'org.apache.curator.framework.recipes.shared.SharedCount', 
  'org.apache.curator.framework.recipes.shared.VersionedValue', 
  'org.apache.curator.retry.ExponentialBackoffRetry', 
  'org.apache.curator.retry.RetryNTimes', 
  'org.apache.curator.utils.CloseableScheduledExecutorService', 
  'org.apache.curator.utils.CloseableUtils', 
  'org.apache.curator.utils.EnsurePath', 
  'org.apache.curator.utils.PathUtils', 
  'org.apache.curator.utils.ThreadUtils', 
  'org.apache.curator.utils.ZKPaths', 
  'org.apache.directory.server.kerberos.shared.keytab.Keytab', 
  'org.apache.directory.server.kerberos.shared.keytab.KeytabEntry', 
  'org.apache.http.NameValuePair', 
  'org.apache.http.client.utils.URIBuilder', 
  'org.apache.http.client.utils.URLEncodedUtils',
  'org.apache.log.Hierarchy', 
  'org.apache.log.Logger', 
  'org.apache.tools.ant.BuildException', 
  'org.apache.tools.ant.DirectoryScanner', 
  'org.apache.tools.ant.Task', 
  'org.apache.tools.ant.taskdefs.Execute', 
  'org.apache.tools.ant.types.FileSet', 
  'org.apache.xml.serialize.OutputFormat', 
  'org.apache.xml.serialize.XMLSerializer', 
  'org.apache.zookeeper.AsyncCallback$StatCallback', 
  'org.apache.zookeeper.AsyncCallback$StringCallback', 
  'org.apache.zookeeper.CreateMode', 
  'org.apache.zookeeper.KeeperException$Code', 
  'org.apache.zookeeper.KeeperException', 
  'org.apache.zookeeper.WatchedEvent', 
  'org.apache.zookeeper.Watcher$Event$EventType', 
  'org.apache.zookeeper.Watcher$Event$KeeperState', 
  'org.apache.zookeeper.Watcher', 
  'org.apache.zookeeper.ZKUtil', 
  'org.apache.zookeeper.ZooDefs$Ids', 
  'org.apache.zookeeper.ZooKeeper', 
  'org.apache.zookeeper.data.ACL', 
  'org.apache.zookeeper.data.Id', 
  'org.apache.zookeeper.data.Stat', 
  'org.codehaus.jackson.JsonEncoding', 
  'org.codehaus.jackson.JsonFactory', 
  'org.codehaus.jackson.JsonGenerator', 
  'org.codehaus.jackson.JsonGenerator$Feature', 
  'org.codehaus.jackson.JsonNode', 
  'org.codehaus.jackson.map.MappingJsonFactory', 
  'org.codehaus.jackson.map.ObjectMapper', 
  'org.codehaus.jackson.map.ObjectReader', 
  'org.codehaus.jackson.map.ObjectWriter', 
  'org.codehaus.jackson.node.ContainerNode', 
  'org.codehaus.jackson.type.TypeReference', 
  'org.codehaus.jackson.util.MinimalPrettyPrinter', 
  'org.fusesource.leveldbjni.JniDBFactory', 
  'org.iq80.leveldb.DB', 
  'org.iq80.leveldb.Options', 
  'org.iq80.leveldb.WriteBatch', 
  'org.mortbay.jetty.Connector', 
  'org.mortbay.jetty.Handler', 
  'org.mortbay.jetty.InclusiveByteRange', 
  'org.mortbay.jetty.MimeTypes', 
  'org.mortbay.jetty.NCSARequestLog', 
  'org.mortbay.jetty.RequestLog', 
  'org.mortbay.jetty.Server', 
  'org.mortbay.jetty.handler.ContextHandler$SContext', 
  'org.mortbay.jetty.handler.ContextHandler', 
  'org.mortbay.jetty.handler.ContextHandlerCollection', 
  'org.mortbay.jetty.handler.HandlerCollection', 
  'org.mortbay.jetty.handler.RequestLogHandler', 
  'org.mortbay.jetty.nio.SelectChannelConnector', 
  'org.mortbay.jetty.security.SslSocketConnector', 
  'org.mortbay.jetty.servlet.AbstractSessionManager', 
  'org.mortbay.jetty.servlet.Context', 
  'org.mortbay.jetty.servlet.DefaultServlet', 
  'org.mortbay.jetty.servlet.FilterHolder', 
  'org.mortbay.jetty.servlet.FilterMapping', 
  'org.mortbay.jetty.servlet.ServletHandler', 
  'org.mortbay.jetty.servlet.ServletHolder', 
  'org.mortbay.jetty.servlet.SessionHandler', 
  'org.mortbay.jetty.webapp.WebAppContext', 
  'org.mortbay.log.Log', 
  'org.mortbay.thread.QueuedThreadPool', 
  'org.mortbay.util.MultiException', 
  'org.mortbay.util.ajax.JSON$Convertible', 
  'org.mortbay.util.ajax.JSON$Output', 
  'org.mortbay.util.ajax.JSON', 
  'org.znerd.xmlenc.XMLOutputter',

  // internal java api: sun.net.dns.ResolverConfiguration
  // internal java api: sun.net.util.IPAddressUtil
  'org.apache.hadoop.security.SecurityUtil$QualifiedHostResolver',

  // internal java api: sun.misc.Unsafe
  'com.google.common.cache.Striped64',
  'com.google.common.cache.Striped64$1',
  'com.google.common.cache.Striped64$Cell',
  'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', 
  'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
  'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
  'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1',
  'org.apache.hadoop.io.nativeio.NativeIO',
  'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm',
  'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot',

  // internal java api: sun.nio.ch.DirectBuffer
  // internal java api: sun.misc.Cleaner
  'org.apache.hadoop.io.nativeio.NativeIO$POSIX',
  'org.apache.hadoop.crypto.CryptoStreamUtils',
 
  // internal java api: sun.misc.SignalHandler
  'org.apache.hadoop.util.SignalLogger$Handler',
]