mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-19 03:15:15 +00:00
371cb9a8ce
We had this as a dependency for legacy dependencies that still needed the Log4j 1.2 API. This appears to no longer be necessary, so this commit removes this artifact as a dependency. To remove this dependency, we had to fix a few places where we were accidentally relying on Log4j 1.2 instead of Log4j 2 (easy to do, since both APIs were on the compile-time classpath). Finally, we can remove our custom Netty logger factory. This was needed when we were on Log4j 1.2 and handled logging in our own unique way. When we migrated to Log4j 2 we could have dropped this dependency. However, even then Netty would still pick up Log4j 1.2 since it was on the classpath, thus the advantage to removing this as a dependency now.
258 lines
10 KiB
Groovy
258 lines
10 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
|
|
import java.nio.file.Files
|
|
import java.nio.file.Path
|
|
import java.nio.file.Paths
|
|
|
|
apply plugin: 'elasticsearch.test.fixtures'
|
|
|
|
esplugin {
|
|
description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
|
|
classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'hadoop2': '2.8.1'
|
|
]
|
|
|
|
testFixtures.useFixture ":test:fixtures:krb5kdc-fixture"
|
|
|
|
configurations {
|
|
hdfsFixture
|
|
}
|
|
|
|
dependencies {
|
|
compile "org.apache.hadoop:hadoop-client:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-common:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-auth:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
|
|
compile "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop2}"
|
|
compile 'org.apache.htrace:htrace-core4:4.0.1-incubating'
|
|
compile 'com.google.guava:guava:11.0.2'
|
|
compile 'com.google.protobuf:protobuf-java:2.5.0'
|
|
compile 'commons-logging:commons-logging:1.1.3'
|
|
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
|
|
compile 'commons-cli:commons-cli:1.2'
|
|
compile "commons-codec:commons-codec:${versions.commonscodec}"
|
|
compile 'commons-collections:commons-collections:3.2.2'
|
|
compile 'commons-configuration:commons-configuration:1.6'
|
|
compile 'commons-io:commons-io:2.4'
|
|
compile 'commons-lang:commons-lang:2.6'
|
|
compile 'javax.servlet:servlet-api:2.5'
|
|
compile "org.slf4j:slf4j-api:${versions.slf4j}"
|
|
compile "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
|
|
|
|
hdfsFixture project(':test:fixtures:hdfs-fixture')
|
|
// Set the keytab files in the classpath so that we can access them from test code without the security manager
|
|
// freaking out.
|
|
testRuntime fileTree(dir: project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab").parent, include: ['*.keytab'])
|
|
}
|
|
|
|
dependencyLicenses {
|
|
mapping from: /hadoop-.*/, to: 'hadoop'
|
|
}
|
|
|
|
String realm = "BUILD.ELASTIC.CO"
|
|
String krb5conf = project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")
|
|
|
|
// Create HDFS File System Testing Fixtures for HA/Secure combinations
|
|
for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) {
|
|
project.tasks.create(fixtureName, org.elasticsearch.gradle.test.AntFixture) {
|
|
dependsOn project.configurations.hdfsFixture, project(':test:fixtures:krb5kdc-fixture').tasks.postProcessFixture
|
|
executable = new File(project.runtimeJavaHome, 'bin/java')
|
|
env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
|
|
onlyIf { project(':test:fixtures:krb5kdc-fixture').buildFixture.enabled }
|
|
waitCondition = { fixture, ant ->
|
|
// the hdfs.MiniHDFS fixture writes the ports file when
|
|
// it's ready, so we can just wait for the file to exist
|
|
return fixture.portsFile.exists()
|
|
}
|
|
|
|
final List<String> miniHDFSArgs = []
|
|
|
|
// If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options
|
|
if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
|
|
miniHDFSArgs.add("-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")}");
|
|
rootProject.globalInfo.ready {
|
|
if (project.runtimeJavaVersion == JavaVersion.VERSION_1_9) {
|
|
miniHDFSArgs.add('--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED')
|
|
}
|
|
}
|
|
}
|
|
// If it's an HA fixture, set a nameservice to use in the JVM options
|
|
if (fixtureName.equals('haHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
|
|
miniHDFSArgs.add("-Dha-nameservice=ha-hdfs")
|
|
}
|
|
|
|
// Common options
|
|
miniHDFSArgs.add('hdfs.MiniHDFS')
|
|
miniHDFSArgs.add(baseDir)
|
|
|
|
// If it's a secure fixture, then set the principal name and keytab locations to use for auth.
|
|
if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
|
|
miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}")
|
|
miniHDFSArgs.add(
|
|
project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab")
|
|
)
|
|
}
|
|
|
|
args miniHDFSArgs.toArray()
|
|
}
|
|
}
|
|
|
|
for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) {
|
|
task "${integTestTaskName}"(type: RestIntegTestTask) {
|
|
description = "Runs rest tests against an elasticsearch cluster with HDFS."
|
|
dependsOn(project.bundlePlugin)
|
|
runner {
|
|
if (integTestTaskName.contains("Secure")) {
|
|
dependsOn secureHdfsFixture
|
|
systemProperty "test.krb5.principal.es", "elasticsearch@${realm}"
|
|
systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}"
|
|
jvmArgs "-Djava.security.krb5.conf=${krb5conf}"
|
|
systemProperty (
|
|
"test.krb5.keytab.hdfs",
|
|
project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab")
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
testClusters."${integTestTaskName}" {
|
|
plugin(file(bundlePlugin.archiveFile))
|
|
if (integTestTaskName.contains("Secure")) {
|
|
systemProperty "java.security.krb5.conf", krb5conf
|
|
extraConfigFile(
|
|
"repository-hdfs/krb5.keytab",
|
|
file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}")
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// Determine HDFS Fixture compatibility for the current build environment.
|
|
boolean fixtureSupported = false
|
|
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
|
// hdfs fixture will not start without hadoop native libraries on windows
|
|
String nativePath = System.getenv("HADOOP_HOME")
|
|
if (nativePath != null) {
|
|
Path path = Paths.get(nativePath);
|
|
if (Files.isDirectory(path) &&
|
|
Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
|
|
Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
|
|
Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
|
|
fixtureSupported = true
|
|
} else {
|
|
throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin");
|
|
}
|
|
}
|
|
} else {
|
|
fixtureSupported = true
|
|
}
|
|
|
|
boolean legalPath = rootProject.rootDir.toString().contains(" ") == false
|
|
if (legalPath == false) {
|
|
fixtureSupported = false
|
|
}
|
|
|
|
// Always ignore HA integration tests in the normal integration test runner, they are included below as
|
|
// part of their own HA-specific integration test tasks.
|
|
integTest.runner {
|
|
exclude('**/Ha*TestSuiteIT.class')
|
|
}
|
|
|
|
if (fixtureSupported) {
|
|
// Check depends on the HA test. Already depends on the standard test.
|
|
project.check.dependsOn(integTestHa)
|
|
|
|
// Both standard and HA tests depend on their respective HDFS fixtures
|
|
integTest.dependsOn hdfsFixture
|
|
integTestHa.dependsOn haHdfsFixture
|
|
|
|
// The normal test runner only runs the standard hdfs rest tests
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.suite', 'hdfs_repository'
|
|
}
|
|
|
|
// Only include the HA integration tests for the HA test task
|
|
integTestHa.runner {
|
|
setIncludes(['**/Ha*TestSuiteIT.class'])
|
|
}
|
|
} else {
|
|
if (legalPath) {
|
|
logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
|
|
} else {
|
|
logger.warn("hdfsFixture unsupported since there are spaces in the path: '" + rootProject.rootDir.toString() + "'")
|
|
}
|
|
|
|
// The normal integration test runner will just test that the plugin loads
|
|
integTest.runner {
|
|
systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
|
|
}
|
|
// HA fixture is unsupported. Don't run them.
|
|
integTestHa.setEnabled(false)
|
|
}
|
|
|
|
check.dependsOn(integTestSecure, integTestSecureHa)
|
|
|
|
// Run just the secure hdfs rest test suite.
|
|
integTestSecure.runner {
|
|
systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
|
|
}
|
|
// Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner.
|
|
integTestSecure.runner {
|
|
exclude('**/Ha*TestSuiteIT.class')
|
|
}
|
|
// Only include the HA integration tests for the HA test task
|
|
integTestSecureHa.runner {
|
|
setIncludes(['**/Ha*TestSuiteIT.class'])
|
|
}
|
|
|
|
thirdPartyAudit {
|
|
ignoreMissingClasses()
|
|
ignoreViolations (
|
|
// internal java api: sun.net.dns.ResolverConfiguration
|
|
// internal java api: sun.net.util.IPAddressUtil
|
|
'org.apache.hadoop.security.SecurityUtil$QualifiedHostResolver',
|
|
|
|
// internal java api: sun.misc.Unsafe
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
|
|
'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1',
|
|
'org.apache.hadoop.io.nativeio.NativeIO',
|
|
'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm',
|
|
'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot',
|
|
|
|
// internal java api: sun.nio.ch.DirectBuffer
|
|
// internal java api: sun.misc.Cleaner
|
|
'org.apache.hadoop.io.nativeio.NativeIO$POSIX',
|
|
'org.apache.hadoop.crypto.CryptoStreamUtils',
|
|
|
|
// internal java api: sun.misc.SignalHandler
|
|
'org.apache.hadoop.util.SignalLogger$Handler',
|
|
)
|
|
}
|