honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add JDK11 support and enable in CI (#31644) 

* Upgrade bouncycastle

Required to fix
`bcprov-jdk15on-1.55.jar; invalid manifest format `
on jdk 11

* Downgrade bouncycastle to avoid invalid manifest

* Add checksum for new jars

* Update tika permissions for jdk 11

* Mute test failing on jdk 11

* Add JDK11 to CI

* Thread#stop(Throwable) was removed

http://mail.openjdk.java.net/pipermail/core-libs-dev/2018-June/053536.html

* Disable failing tests #31456

* Temprorarily disable doc tests

To see if there are other failures on JDK11

* Only blacklist specific doc tests

* Disable only failing tests in ingest attachment plugin

* Mute failing HDFS tests #31498

* Mute failing lang-painless tests #31500

* Fix backwards compatability builds

Fix JAVA version to 10 for ES 6.3

* Add 6.x to bwx -> java10

* Prefix out and err from buildBwcVersion for readability

```
> Task :distribution:bwc:next-bugfix-snapshot:buildBwcVersion
  [bwc] :buildSrc:compileJava
  [bwc] WARNING: An illegal reflective access operation has occurred
  [bwc] WARNING: Illegal reflective access by org.codehaus.groovy.reflection.CachedClass (file:/home/alpar/.gradle/wrapper/dists/gradle-4.5-all/cg9lyzfg3iwv6fa00os9gcgj4/gradle-4.5/lib/groovy-all-2.4.12.jar) to method java.lang.Object.finalize()
  [bwc] WARNING: Please consider reporting this to the maintainers of org.codehaus.groovy.reflection.CachedClass
  [bwc] WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
  [bwc] WARNING: All illegal access operations will be denied in a future release
  [bwc] :buildSrc:compileGroovy
  [bwc] :buildSrc:writeVersionProperties
  [bwc] :buildSrc:processResources
  [bwc] :buildSrc:classes
  [bwc] :buildSrc:jar

```

* Also set RUNTIME_JAVA_HOME for bwcBuild

So that we can make sure it's not too new for the build to understand.

* Align bouncycastle dependency

* fix painles array tets

closes #31500

* Update jar checksums

* Keep 8/10 runtime/compile untill consensus builds on 11

* Only skip failing tests if running on Java 11

* Failures are dependent of compile java version not runtime

* Condition doc test exceptions on compiler java version as well

* Disable hdfs tests based on runtime java

* Set runtime java to minimum supported for bwc

* PR review

* Add comment with ticket for forbidden apis
This commit is contained in:
Alpar Torok 2018-07-05 03:24:01 +00:00 committed by GitHub
parent 0a2ef59c41
commit cf2295b408
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 107 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/matrix-build-javas.yml
View File

@ -7,3 +7,4 @@
ES_BUILD_JAVA:
- java10
- java11

1
.ci/matrix-runtime-javas.yml
View File

@ -8,3 +8,4 @@
ES_RUNTIME_JAVA:
- java8
- java10
- java11

3
buildSrc/src/main/resources/forbidden/jdk-signatures.txt
View File

@ -88,7 +88,8 @@ java.lang.Thread#getAllStackTraces()
@defaultMessage Stopping threads explicitly leads to inconsistent states. Use interrupt() instead.
java.lang.Thread#stop()
java.lang.Thread#stop(java.lang.Throwable)
# uncomment when https://github.com/elastic/elasticsearch/issues/31715 is fixed
# java.lang.Thread#stop(java.lang.Throwable)
@defaultMessage Please do not terminate the application
java.lang.System#exit(int)

36
distribution/bwc/build.gradle
View File

@ -18,12 +18,14 @@
*/
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
import java.nio.charset.StandardCharsets
import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
/**
* This is a dummy project which does a local checkout of the previous
* wire compat version's branch, and builds a snapshot. This allows backcompat
@ -147,12 +149,16 @@ subprojects {
task buildBwcVersion(type: Exec) {
dependsOn checkoutBwcBranch, writeBuildMetadata
// send RUNTIME_JAVA_HOME so the build doesn't fails on newer version the branch doesn't know about
environment('RUNTIME_JAVA_HOME', getJavaHome(it, rootProject.ext.minimumRuntimeVersion.getMajorVersion() as int))
workingDir = checkoutDir
// we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
if (["5.6", "6.0", "6.1"].contains(bwcBranch)) {
// we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
environment('JAVA_HOME', getJavaHome(it, 8))
} else if ("6.2".equals(bwcBranch)) {
environment('JAVA_HOME', getJavaHome(it, 9))
} else if (["6.3", "6.x"].contains(bwcBranch)) {
environment('JAVA_HOME', getJavaHome(it, 10))
} else {
environment('JAVA_HOME', project.compilerJavaHome)
}
@ -177,6 +183,8 @@ subprojects {
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
args "--full-stacktrace"
}
standardOutput = new IndentingOutputStream(System.out)
errorOutput = new IndentingOutputStream(System.err)
doLast {
List missing = artifactFiles.grep { file ->
false == file.exists()
@ -196,3 +204,27 @@ subprojects {
}
}
}
class IndentingOutputStream extends OutputStream {
public static final byte[] INDENT = " [bwc] ".getBytes(StandardCharsets.UTF_8)
private final OutputStream delegate
public IndentingOutputStream(OutputStream delegate) {
this.delegate = delegate
}
@Override
public void write(int b) {
write([b] as int[], 0, 1)
}
public void write(int[] bytes, int offset, int length) {
for (int i = 0; i < bytes.length; i++) {
delegate.write(bytes[i])
if (bytes[i] == '\n') {
delegate.write(INDENT)
}
}
}
}

9
docs/build.gradle
View File

@ -39,6 +39,15 @@ integTestCluster {
setting 'reindex.remote.whitelist', '127.0.0.1:*'
}
// remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed
if (rootProject.ext.compilerJavaVersion.isJava11()) {
integTestRunner {
systemProperty 'tests.rest.blacklist', [
'plugins/ingest-attachment/line_164',
'plugins/ingest-attachment/line_117'
].join(',')
}
}
// Build the cluster with all plugins
project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->

7
modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.painless;
import org.apache.lucene.util.Constants;
import org.elasticsearch.bootstrap.JavaVersion;
import org.hamcrest.Matcher;
import java.lang.invoke.MethodHandle;
@ -41,7 +42,11 @@ public class ArrayTests extends ArrayLikeObjectTestCase {
@Override
protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
return equalTo(Integer.toString(index));
if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
return equalTo(Integer.toString(index));
} else{
return equalTo("Index " + Integer.toString(index) + " out of bounds for length " + Integer.toString(size));
}
}
public void testArrayLengthHelper() throws Throwable {

12
plugins/ingest-attachment/build.gradle
View File

@ -25,11 +25,21 @@ esplugin {
versions << [
'tika': '1.18',
'pdfbox': '2.0.9',
'bouncycastle': '1.55',
'bouncycastle': '1.59',
'poi': '3.17',
'mime4j': '0.8.1'
]
if (rootProject.ext.compilerJavaVersion.isJava11()) {
// disabled until https://github.com/elastic/elasticsearch/issues/31456 is fixed.
integTestRunner {
systemProperty 'tests.rest.blacklist', [
'ingest_attachment/20_attachment_processor/Test indexed chars are configurable',
'ingest_attachment/20_attachment_processor/Test indexed chars are configurable per document'
].join(',')
}
}
dependencies {
// mandatory for tika
compile "org.apache.tika:tika-core:${versions.tika}"

1
plugins/ingest-attachment/licenses/bcmail-jdk15on-1.55.jar.sha1
View File

@ -1 +0,0 @@
5cea2dada69b98698ea975a5c1dd3c91ac8ffbb6

1
plugins/ingest-attachment/licenses/bcmail-jdk15on-1.59.jar.sha1 Normal file
View File

@ -0,0 +1 @@
db389ade95f48592908a84e7050a691c8834723c

1
plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.55.jar.sha1
View File

@ -1 +0,0 @@
6392d8cba22b722c6570d660ca0b3921ff1bae4f

1
plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.59.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9cef0aab8a4bb849a8476c058ce3ff302aba3fff

1
plugins/ingest-attachment/licenses/bcprov-jdk15on-1.55.jar.sha1
View File

@ -1 +0,0 @@
935f2e57a00ec2c489cbd2ad830d4a399708f979

1
plugins/ingest-attachment/licenses/bcprov-jdk15on-1.59.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2507204241ab450456bdb8e8c0a8f986e418bd99

11
plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
View File

@ -164,12 +164,11 @@ final class TikaImpl {
perms.add(new RuntimePermission("getClassLoader"));
// ZipFile needs accessDeclaredMembers on JDK 10; cf. https://bugs.openjdk.java.net/browse/JDK-8187485
if (JavaVersion.current().compareTo(JavaVersion.parse("10")) >= 0) {
/*
* See if this permission can be removed in JDK 11, bump the version here to 12 if not. If this permission can be removed, also
* remove the grant in the plugin-security.policy.
*/
assert JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0;
perms.add(new RuntimePermission("accessDeclaredMembers"));
if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
// TODO remove this and from plugin-security.policy when JDK 11 is the only one we support
// this is needed pre 11, but it's fixed in 11 : https://bugs.openjdk.java.net/browse/JDK-8187485
perms.add(new RuntimePermission("accessDeclaredMembers"));
}
}
perms.setReadOnly();
return perms;

2
plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.ingest.attachment;
import org.apache.commons.io.IOUtils;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.bootstrap.JavaVersion;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.RandomDocumentPicks;
@ -296,6 +297,7 @@ public class AttachmentProcessorTests extends ESTestCase {
}
public void testIndexedChars() throws Exception {
assumeFalse("https://github.com/elastic/elasticsearch/issues/31305", JavaVersion.current().equals(JavaVersion.parse("11")));
processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field",
"target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 19, false, null);

21
plugins/repository-hdfs/build.gradle
View File

@ -1,4 +1,4 @@
/*
1/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
@ -214,6 +214,25 @@ RestIntegTestTask integTestSecureHa = project.tasks.create('integTestSecureHa',
description = "Runs rest tests against an elasticsearch cluster with HDFS configured with HA Namenode and secured by MIT Kerberos."
}
if (rootProject.ext.compilerJavaVersion.isJava11()) {
// TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
integTestRunner {
systemProperty 'tests.rest.blacklist', [
'hdfs_repository/30_snapshot/take snapshot',
'hdfs_repository/40_restore/Create a snapshot and then restore it',
'hdfs_repository/20_repository_verify/HDFS Repository Verify',
'hdfs_repository/30_snapshot_get/Get a snapshot',
'hdfs_repository/20_repository_create/HDFS Repository Creation',
'hdfs_repository/20_repository_delete/HDFS Delete Repository',
'hdfs_repository/30_snapshot_readonly/Get a snapshot - readonly',
].join(',')
}
}
if (rootProject.ext.runtimeJavaVersion.isJava11()) {
// TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
integTestHa.enabled = false
}
// Determine HDFS Fixture compatibility for the current build environment.
boolean fixtureSupported = false
if (Os.isFamily(Os.FAMILY_WINDOWS)) {

12
plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
View File

@ -18,15 +18,11 @@
*/
package org.elasticsearch.repositories.hdfs;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import java.util.Collection;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.elasticsearch.bootstrap.JavaVersion;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.settings.Settings;
@ -35,6 +31,11 @@ import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.util.Collection;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
@ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class})
public class HdfsTests extends ESSingleNodeTestCase {
@ -44,6 +45,7 @@ public class HdfsTests extends ESSingleNodeTestCase {
}
public void testSimpleWorkflow() {
assumeFalse("https://github.com/elastic/elasticsearch/issues/31498", JavaVersion.current().equals(JavaVersion.parse("11")));
Client client = client();
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")

4
x-pack/plugin/core/build.gradle
View File

@ -35,8 +35,8 @@ dependencies {
// security deps
compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
compile 'org.bouncycastle:bcprov-jdk15on:1.58'
compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
compile 'org.bouncycastle:bcprov-jdk15on:1.59'
compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
compile project(path: ':modules:transport-netty4', configuration: 'runtime')
testCompile 'org.elasticsearch:securemock:1.2'

1
x-pack/plugin/core/licenses/bcpkix-jdk15on-1.58.jar.sha1
View File

@ -1 +0,0 @@
15a760a039b040e767a75c77ffcc4ff62558f903

1
x-pack/plugin/core/licenses/bcpkix-jdk15on-1.59.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9cef0aab8a4bb849a8476c058ce3ff302aba3fff

1
x-pack/plugin/core/licenses/bcprov-jdk15on-1.58.jar.sha1
View File

@ -1 +0,0 @@
2c9aa1c4e3372b447ba5daabade4adf2a2264b12

1
x-pack/plugin/core/licenses/bcprov-jdk15on-1.59.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2507204241ab450456bdb8e8c0a8f986e418bd99

4
x-pack/plugin/security/build.gradle
View File

@ -22,8 +22,8 @@ dependencies {
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
compile 'org.bouncycastle:bcprov-jdk15on:1.58'
compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
compile 'org.bouncycastle:bcprov-jdk15on:1.59'
compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
// the following are all SAML dependencies - might as well download the whole internet
compile "org.opensaml:opensaml-core:3.3.0"