mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-04 01:49:15 +00:00
6692e42d9a
This fixes the `lenient` parameter to be `missingClasses`. I will remove this boolean and we can handle them via the normal whitelist. It also adds a check for sheisty classes (jar hell with the jdk). This is inspired by the lucene "sheisty" classes check, but it has false positives. This check is more evil, it validates every class file against the extension classloader as a resource, to see if it exists there. If so: jar hell. This jar hell is a problem for several reasons: 1. causes insanely-hard-to-debug problems (like bugs in forbidden-apis) 2. hides problems (like internal api access) 3. the code you think is executing, is not really executing 4. security permissions are not what you think they are 5. brings in unnecessary dependencies 6. its jar hell The more difficult problems are stuff like jython, where these classes are simply 'uberjared' directly in, so you cant just fix them by removing a bogus dependency. And there is a legit reason for them to do that, they want to support java 1.4.
207 lines
6.2 KiB
Groovy
207 lines
6.2 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
//apply plugin: 'nebula.provided-base'
|
|
|
|
esplugin {
|
|
description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
|
|
classname 'org.elasticsearch.plugin.hadoop.hdfs.HdfsPlugin'
|
|
}
|
|
|
|
configurations {
|
|
hadoop1
|
|
hadoop2
|
|
}
|
|
|
|
versions << [
|
|
'hadoop1': '1.2.1',
|
|
'hadoop2': '2.7.1'
|
|
]
|
|
|
|
dependencies {
|
|
provided "org.elasticsearch:elasticsearch:${versions.elasticsearch}"
|
|
provided "org.apache.hadoop:hadoop-core:${versions.hadoop1}"
|
|
|
|
// use Hadoop1 to compile and test things (a subset of Hadoop2)
|
|
testCompile "org.apache.hadoop:hadoop-core:${versions.hadoop1}"
|
|
testCompile "org.apache.hadoop:hadoop-test:${versions.hadoop1}"
|
|
// Hadoop dependencies
|
|
testCompile "commons-configuration:commons-configuration:1.6"
|
|
testCompile "commons-lang:commons-lang:${versions.commonslang}"
|
|
testCompile "commons-collections:commons-collections:3.2.2"
|
|
testCompile "commons-net:commons-net:1.4.1"
|
|
testCompile "org.mortbay.jetty:jetty:6.1.26"
|
|
testCompile "org.mortbay.jetty:jetty-util:6.1.26"
|
|
testCompile "org.mortbay.jetty:servlet-api:2.5-20081211"
|
|
testCompile "com.sun.jersey:jersey-core:1.8"
|
|
|
|
|
|
hadoop1("org.apache.hadoop:hadoop-core:${versions.hadoop1}") {
|
|
exclude module: "commons-cli"
|
|
exclude group: "com.sun.jersey"
|
|
exclude group: "org.mortbay.jetty"
|
|
exclude group: "tomcat"
|
|
exclude module: "commons-el"
|
|
exclude module: "hsqldb"
|
|
exclude group: "org.eclipse.jdt"
|
|
exclude module: "commons-beanutils"
|
|
exclude module: "commons-beanutils-core"
|
|
exclude module: "junit"
|
|
// provided by ES itself
|
|
exclude group: "log4j"
|
|
}
|
|
|
|
hadoop2("org.apache.hadoop:hadoop-client:${versions.hadoop2}") {
|
|
exclude module: "commons-cli"
|
|
exclude group: "com.sun.jersey"
|
|
exclude group: "com.sun.jersey.contribs"
|
|
exclude group: "com.sun.jersey.jersey-test-framework"
|
|
exclude module: "guice"
|
|
exclude group: "org.mortbay.jetty"
|
|
exclude group: "tomcat"
|
|
exclude module: "commons-el"
|
|
exclude module: "hsqldb"
|
|
exclude group: "org.eclipse.jdt"
|
|
exclude module: "commons-beanutils"
|
|
exclude module: "commons-beanutils-core"
|
|
exclude module: "javax.servlet"
|
|
exclude module: "junit"
|
|
// provided by ES itself
|
|
exclude group: "log4j"
|
|
}
|
|
|
|
hadoop2("org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}") {
|
|
exclude module: "guava"
|
|
exclude module: "junit"
|
|
// provided by ES itself
|
|
exclude group: "log4j"
|
|
}
|
|
}
|
|
|
|
configurations.all {
|
|
resolutionStrategy {
|
|
force "commons-codec:commons-codec:${versions.commonscodec}"
|
|
force "commons-logging:commons-logging:${versions.commonslogging}"
|
|
force "commons-lang:commons-lang:2.6"
|
|
force "commons-httpclient:commons-httpclient:3.0.1"
|
|
force "org.codehaus.jackson:jackson-core-asl:1.8.8"
|
|
force "org.codehaus.jackson:jackson-mapper-asl:1.8.8"
|
|
force "com.google.code.findbugs:jsr305:3.0.0"
|
|
force "com.google.guava:guava:16.0.1"
|
|
force "org.slf4j:slf4j-api:1.7.10"
|
|
force "org.slf4j:slf4j-log4j12:1.7.10"
|
|
}
|
|
}
|
|
|
|
|
|
dependencyLicenses {
|
|
mapping from: /hadoop-core.*/, to: 'hadoop-1'
|
|
mapping from: /hadoop-.*/, to: 'hadoop-2'
|
|
}
|
|
|
|
compileJava.options.compilerArgs << '-Xlint:-deprecation,-rawtypes'
|
|
|
|
// main jar includes just the plugin classes
|
|
jar {
|
|
include "org/elasticsearch/plugin/hadoop/hdfs/*"
|
|
}
|
|
|
|
// hadoop jar (which actually depend on Hadoop)
|
|
task hadoopLinkedJar(type: Jar, dependsOn:jar) {
|
|
appendix "internal"
|
|
from sourceSets.main.output.classesDir
|
|
// exclude plugin
|
|
exclude "org/elasticsearch/plugin/hadoop/hdfs/*"
|
|
}
|
|
|
|
|
|
bundlePlugin.dependsOn hadoopLinkedJar
|
|
|
|
// configure 'bundle' as being w/o Hadoop deps
|
|
bundlePlugin {
|
|
into ("internal-libs") {
|
|
from hadoopLinkedJar.archivePath
|
|
}
|
|
|
|
into ("hadoop-libs") {
|
|
from configurations.hadoop2.allArtifacts.files
|
|
from configurations.hadoop2
|
|
}
|
|
}
|
|
|
|
|
|
task distZipHadoop1(type: Zip, dependsOn: [hadoopLinkedJar, jar]) { zipTask ->
|
|
from (zipTree(bundlePlugin.archivePath)) {
|
|
include "*"
|
|
include "internal-libs/**"
|
|
}
|
|
|
|
description = "Builds archive (with Hadoop1 dependencies) suitable for download page."
|
|
classifier = "hadoop1"
|
|
|
|
into ("hadoop-libs") {
|
|
from configurations.hadoop1.allArtifacts.files
|
|
from configurations.hadoop1
|
|
}
|
|
}
|
|
|
|
task distZipHadoop2(type: Zip, dependsOn: [hadoopLinkedJar, jar]) { zipTask ->
|
|
from (zipTree(bundlePlugin.archivePath)) {
|
|
include "*"
|
|
include "internal-libs/**"
|
|
}
|
|
|
|
description = "Builds archive (with Hadoop2/YARN dependencies) suitable for download page."
|
|
classifier = "hadoop2"
|
|
|
|
into ("hadoop-libs") {
|
|
from configurations.hadoop2.allArtifacts.files
|
|
from configurations.hadoop2
|
|
}
|
|
}
|
|
|
|
task distZipNoHadoop(type: Zip, dependsOn: [hadoopLinkedJar, jar]) { zipTask ->
|
|
from (zipTree(bundlePlugin.archivePath)) {
|
|
exclude "hadoop-libs/**"
|
|
}
|
|
|
|
from sourceSets.main.output.resourcesDir
|
|
|
|
description = "Builds archive (without any Hadoop dependencies) suitable for download page."
|
|
classifier = "lite"
|
|
}
|
|
|
|
|
|
artifacts {
|
|
archives bundlePlugin
|
|
'default' bundlePlugin
|
|
archives distZipHadoop1
|
|
archives distZipHadoop2
|
|
archives distZipNoHadoop
|
|
}
|
|
|
|
integTest {
|
|
cluster {
|
|
plugin(pluginProperties.extension.name, zipTree(distZipHadoop2.archivePath))
|
|
}
|
|
}
|
|
|
|
// classes are missing, e.g. org.mockito.Mockito
|
|
thirdPartyAudit.missingClasses = true
|