honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove guava from transitive compile classpath (#54309) (#54695) 

Guava was removed from Elasticsearch many years ago, but remnants of it
remain due to transitive dependencies. When a dependency pulls guava
into the compile classpath, devs can inadvertently begin using methods
from guava without realizing it. This commit moves guava to a runtime
dependency in the modules that it is needed.

Note that one special case is the html sanitizer in watcher. The third
party dep uses guava in the PolicyFactory class signature. However, only
calling a method on the PolicyFactory actually causes the class to be
loaded, a reference alone does not trigger compilation to look at the
class implementation. There we utilize a MethodHandle for invoking the
relevant method at runtime, where guava will continue to exist.
This commit is contained in:
Ryan Ernst 2020-04-07 23:20:17 -07:00 committed by GitHub
parent ca20b8a828
commit 37795d259a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 168 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.gradle
View File

@ -45,6 +45,7 @@ apply from: 'gradle/build-scan.gradle'
apply from: 'gradle/build-complete.gradle'
apply from: 'gradle/runtime-jdk-provision.gradle'
apply from: 'gradle/ide.gradle'
apply from: 'gradle/forbidden-dependencies.gradle'
apply from: 'gradle/formatting.gradle'
// common maven publishing configuration

1
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -301,6 +301,7 @@ class BuildPlugin implements Plugin<Project> {
project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
}
/** Adds repositories used by ES dependencies */

5
buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
View File

@ -134,7 +134,8 @@ class PluginBuildPlugin implements Plugin<Project> {
}
createIntegTestTask(project)
createBundleTasks(project, extension)
project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
project.configurations.getByName('default')
.extendsFrom(project.configurations.getByName('runtimeClasspath'))
// allow running ES with this plugin in the foreground of a build
project.tasks.register('run', RunTask) {
dependsOn(project.tasks.bundlePlugin)
@ -227,7 +228,7 @@ class PluginBuildPlugin implements Plugin<Project> {
* that shadow jar.
*/
from { project.plugins.hasPlugin(ShadowPlugin) ? project.shadowJar : project.jar }
from project.configurations.runtime - project.configurations.compileOnly
from project.configurations.runtimeClasspath - project.configurations.compileOnly
// extra files for the plugin to go into the zip
from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging
from('src/main') {

14
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View File

@ -28,6 +28,7 @@ import org.elasticsearch.gradle.util.Util
import org.gradle.api.JavaVersion
import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.plugins.quality.Checkstyle
import org.gradle.api.tasks.TaskProvider
@ -142,6 +143,19 @@ class PrecommitTasks {
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
project.tasks.withType(CheckForbiddenApis).configureEach {
dependsOn(buildResources)
// use the runtime classpath if we have it, but some qa projects don't have one...
if (name.endsWith('Test')) {
FileCollection runtime = project.sourceSets.test.runtimeClasspath
if (runtime != null) {
classpath = runtime.plus(project.sourceSets.test.compileClasspath)
}
} else {
FileCollection runtime = project.sourceSets.main.runtimeClasspath
if (runtime != null) {
classpath = runtime.plus(project.sourceSets.main.compileClasspath)
}
}
targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
project.logger.warn(

2
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java
View File

@ -388,7 +388,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
}
private Configuration getRuntimeConfiguration() {
Configuration runtime = getProject().getConfigurations().findByName("runtime");
Configuration runtime = getProject().getConfigurations().findByName("runtimeClasspath");
if (runtime == null) {
return getProject().getConfigurations().getByName("testCompile");
}

2
distribution/tools/keystore-cli/build.gradle
View File

@ -24,5 +24,5 @@ dependencies {
compileOnly project(":libs:elasticsearch-cli")
testCompile project(":test:framework")
testCompile 'com.google.jimfs:jimfs:1.1'
testCompile 'com.google.guava:guava:18.0'
testRuntimeOnly 'com.google.guava:guava:18.0'
}

4
distribution/tools/plugin-cli/build.gradle
View File

@ -1,5 +1,3 @@
import org.elasticsearch.gradle.info.BuildParams
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
@ -30,7 +28,7 @@ dependencies {
compile "org.bouncycastle:bc-fips:1.0.1"
testCompile project(":test:framework")
testCompile 'com.google.jimfs:jimfs:1.1'
testCompile 'com.google.guava:guava:18.0'
testRuntimeOnly 'com.google.guava:guava:18.0'
}
dependencyLicenses {

27
gradle/forbidden-dependencies.gradle Normal file
View File

@ -0,0 +1,27 @@
// we do not want any of these dependencies on the compilation classpath
// because they could then be used within Elasticsearch
List<String> FORBIDDEN_DEPENDENCIES = [
'guava'
]
Closure checkDeps = { Configuration configuration ->
configuration.resolutionStrategy.eachDependency {
String artifactName = it.target.name
if (FORBIDDEN_DEPENDENCIES.contains(artifactName)) {
throw new GradleException("Dependency '${artifactName}' on configuration '${configuration.name}' is not allowed. " +
"If it is needed as a transitive depenency, try adding it to the runtime classpath")
}
}
}
subprojects {
if (project.path.startsWith(':test:fixtures:') || project.path.equals(':build-tools')) {
// fixtures are allowed to use whatever dependencies they want...
return
}
pluginManager.withPlugin('java') {
checkDeps(configurations.compileClasspath)
checkDeps(configurations.testCompileClasspath)
}
}

2
plugins/repository-azure/build.gradle
View File

@ -27,7 +27,7 @@ esplugin {
dependencies {
compile 'com.microsoft.azure:azure-storage:8.6.2'
compile 'com.microsoft.azure:azure-keyvault-core:1.0.0'
compile 'com.google.guava:guava:20.0'
runtimeOnly 'com.google.guava:guava:20.0'
compile 'org.apache.commons:commons-lang3:3.4'
testCompile project(':test:fixtures:azure-fixture')
}

2
plugins/repository-gcs/build.gradle
View File

@ -27,7 +27,7 @@ esplugin {
dependencies {
compile 'com.google.cloud:google-cloud-storage:1.106.0'
compile 'com.google.cloud:google-cloud-core:1.93.3'
compile 'com.google.guava:guava:26.0-jre'
runtimeOnly 'com.google.guava:guava:26.0-jre'
compile 'com.google.http-client:google-http-client:1.34.2'
compile "commons-logging:commons-logging:${versions.commonslogging}"
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"

2
plugins/repository-hdfs/build.gradle
View File

@ -52,7 +52,7 @@ dependencies {
compile "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}"
compile "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop2}"
compile 'org.apache.htrace:htrace-core4:4.0.1-incubating'
compile 'com.google.guava:guava:11.0.2'
runtimeOnly 'com.google.guava:guava:11.0.2'
compile 'com.google.protobuf:protobuf-java:2.5.0'
compile 'commons-logging:commons-logging:1.1.3'
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"

2
server/build.gradle
View File

@ -131,8 +131,6 @@ dependencies {
// tests use the locally compiled version of server
exclude group: 'org.elasticsearch', module: 'server'
}
testCompile 'com.google.guava:guava:18.0'
}
compileJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked"

4
server/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
View File

@ -102,4 +102,8 @@ public class Iterables {
return it.next();
}
}
public static long size(Iterable<?> iterable) {
return StreamSupport.stream(iterable.spliterator(), true).count();
}
}

17
server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.action.fieldcaps;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractSerializingTestCase;
@ -28,6 +26,7 @@ import org.elasticsearch.test.AbstractSerializingTestCase;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@ -101,8 +100,8 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase<FieldCap
builder = new FieldCapabilities.Builder("field", "type");
builder.add("index1", true, true, Collections.emptyMap());
builder.add("index2", true, true, ImmutableMap.of("foo", "bar"));
builder.add("index3", true, true, ImmutableMap.of("foo", "quux"));
builder.add("index2", true, true, Collections.singletonMap("foo", "bar"));
builder.add("index3", true, true, Collections.singletonMap("foo", "quux"));
{
FieldCapabilities cap1 = builder.build(false);
assertThat(cap1.isSearchable(), equalTo(true));
@ -110,7 +109,7 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase<FieldCap
assertNull(cap1.indices());
assertNull(cap1.nonSearchableIndices());
assertNull(cap1.nonAggregatableIndices());
assertEquals(ImmutableMap.of("foo", ImmutableSet.of("bar", "quux")), cap1.meta());
assertEquals(Collections.singletonMap("foo", new HashSet<>(Arrays.asList("bar", "quux"))), cap1.meta());
FieldCapabilities cap2 = builder.build(true);
assertThat(cap2.isSearchable(), equalTo(true));
@ -119,7 +118,7 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase<FieldCap
assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"}));
assertNull(cap2.nonSearchableIndices());
assertNull(cap2.nonAggregatableIndices());
assertEquals(ImmutableMap.of("foo", ImmutableSet.of("bar", "quux")), cap2.meta());
assertEquals(Collections.singletonMap("foo", new HashSet<>(Arrays.asList("bar", "quux"))), cap2.meta());
}
}
@ -152,10 +151,10 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase<FieldCap
meta = Collections.emptyMap();
break;
case 1:
meta = ImmutableMap.of("foo", ImmutableSet.of("bar"));
meta = Collections.singletonMap("foo", Collections.singleton("bar"));
break;
default:
meta = ImmutableMap.of("foo", ImmutableSet.of("bar", "baz"));
meta = Collections.singletonMap("foo", new HashSet<>(Arrays.asList("bar", "baz")));
break;
}
@ -232,7 +231,7 @@ public class FieldCapabilitiesTests extends AbstractSerializingTestCase<FieldCap
case 7:
Map<String, Set<String>> newMeta;
if (meta.isEmpty()) {
newMeta = ImmutableMap.of("foo", ImmutableSet.of("bar"));
newMeta = Collections.singletonMap("foo", Collections.singleton("bar"));
} else {
newMeta = Collections.emptyMap();
}

22
server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@ -219,14 +218,15 @@ public class TypeParsersTests extends ESTestCase {
Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(null, null, null, null, null);
{
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", 3));
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta", 3));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertEquals("[meta] must be an object, got Integer[3] for field [foo]", e.getMessage());
}
{
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", ImmutableMap.of("veryloooooooooooongkey", 3L)));
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta",
Collections.singletonMap("veryloooooooooooongkey", 3L)));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertEquals("[meta] keys can't be longer than 20 chars, but got [veryloooooooooooongkey] for field [foo]",
@ -241,7 +241,7 @@ public class TypeParsersTests extends ESTestCase {
meta.put("foo4", "3");
meta.put("foo5", "3");
meta.put("foo6", "3");
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", meta));
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta", meta));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertEquals("[meta] can't have more than 5 entries, but got 6 on field [foo]",
@ -249,15 +249,19 @@ public class TypeParsersTests extends ESTestCase {
}
{
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", ImmutableMap.of("foo", ImmutableMap.of("bar", "baz"))));
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta",
Collections.singletonMap("foo", Collections.singletonMap("bar", "baz"))));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertEquals("[meta] values can only be strings, but got SingletonImmutableBiMap[{bar=baz}] for field [foo]",
assertEquals("[meta] values can only be strings, but got SingletonMap[{bar=baz}] for field [foo]",
e.getMessage());
}
{
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", ImmutableMap.of("bar", "baz", "foo", 3)));
Map<String, Object> inner = new HashMap<>();
inner.put("bar", "baz");
inner.put("foo", 3);
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta", inner));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertEquals("[meta] values can only be strings, but got Integer[3] for field [foo]",
@ -267,7 +271,7 @@ public class TypeParsersTests extends ESTestCase {
{
Map<String, String> meta = new HashMap<>();
meta.put("foo", null);
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", meta));
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta", meta));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertEquals("[meta] values can't be null (field [foo])",
@ -278,7 +282,7 @@ public class TypeParsersTests extends ESTestCase {
String longString = IntStream.range(0, 51)
.mapToObj(Integer::toString)
.collect(Collectors.joining());
Map<String, Object> mapping = new HashMap<>(ImmutableMap.of("meta", ImmutableMap.of("foo", longString)));
Map<String, Object> mapping = new HashMap<>(Collections.singletonMap("meta", Collections.singletonMap("foo", longString)));
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> TypeParsers.parseField(builder, builder.name, mapping, parserContext));
assertThat(e.getMessage(), Matchers.startsWith("[meta] values can't be longer than 50 chars"));

2
x-pack/plugin/identity-provider/build.gradle
View File

@ -48,7 +48,7 @@ dependencies {
compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
compile "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
compile 'com.google.guava:guava:19.0'
runtimeOnly 'com.google.guava:guava:19.0'
testCompile 'org.elasticsearch:securemock:1.2'
testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"

7
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.ml.integration;
import com.google.common.collect.Ordering;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@ -634,7 +633,11 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
// Assert that all the class probabilities lie within [0, 1] interval.
classProbabilities.forEach(p -> assertThat(p, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))));
// Assert that the top classes are listed in the order of decreasing scores.
assertThat(Ordering.natural().reverse().isOrdered(classScores), is(true));
double prevScore = classScores.get(0);
for (int i = 1; i < classScores.size(); ++i) {
double score = classScores.get(i);
assertThat("class " + i, score, lessThanOrEqualTo(prevScore));
}
}
private <T> void assertEvaluation(String dependentVariable, List<T> dependentVariableValues, String predictedClassField) {

3
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/ProcessResultsParserTests.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.ml.process;
import com.google.common.base.Charsets;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@ -59,7 +58,7 @@ public class ProcessResultsParserTests extends ESTestCase {
public void testParseResults() throws IOException {
String input = "[{\"field_1\": \"a\", \"field_2\": 1.0}, {\"field_1\": \"b\", \"field_2\": 2.0},"
+ " {\"field_1\": \"c\", \"field_2\": 3.0}]";
try (InputStream inputStream = new ByteArrayInputStream(input.getBytes(Charsets.UTF_8))) {
try (InputStream inputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))) {
ProcessResultsParser<TestResult> parser = new ProcessResultsParser<>(TestResult.PARSER, NamedXContentRegistry.EMPTY);
Iterator<TestResult> testResultIterator = parser.parseResults(inputStream);

2
x-pack/plugin/security/build.gradle
View File

@ -55,7 +55,7 @@ dependencies {
compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
compile "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
compile 'com.google.guava:guava:19.0'
runtimeOnly 'com.google.guava:guava:19.0'
// Dependencies for oidc
compile "com.nimbusds:oauth2-oidc-sdk:7.0.2"

6
x-pack/plugin/security/cli/build.gradle
View File

@ -10,7 +10,11 @@ dependencies {
compileOnly project(path: xpackModule('core'), configuration: 'default')
compile "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
testImplementation 'com.google.jimfs:jimfs:1.1'
testImplementation('com.google.jimfs:jimfs:1.1') {
// this is provided by the runtime classpath, from the security project
exclude group: 'com.google.guava', module: 'guava'
}
testRuntimeOnly 'com.google.guava:guava:19.0'
testCompile project(":test:framework")
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
}

2
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.security.authc;
import com.google.common.collect.Sets;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@ -22,6 +21,7 @@ import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.SecurityIntegTestCase;
import org.elasticsearch.test.SecuritySettingsSource;

30
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
View File

@ -5,8 +5,6 @@
*/
package org.elasticsearch.xpack.sql.parser;
import com.google.common.base.Joiner;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ql.expression.Alias;
import org.elasticsearch.xpack.ql.expression.Literal;
@ -214,11 +212,11 @@ public class SqlParserTests extends ESTestCase {
// Create expression in the form of a = b OR a = b OR ... a = b
// 1000 elements is ok
new SqlParser().createExpression(Joiner.on(" OR ").join(nCopies(1000, "a = b")));
new SqlParser().createExpression(join(" OR ", nCopies(1000, "a = b")));
// 5000 elements cause stack overflow
ParsingException e = expectThrows(ParsingException.class, () ->
new SqlParser().createExpression(Joiner.on(" OR ").join(nCopies(5000, "a = b"))));
new SqlParser().createExpression(join(" OR ", nCopies(5000, "a = b"))));
assertThat(e.getMessage(),
startsWith("line -1:0: SQL statement is too large, causing stack overflow when generating the parsing tree: ["));
}
@ -228,11 +226,11 @@ public class SqlParserTests extends ESTestCase {
// 200 elements is ok
new SqlParser().createExpression(
Joiner.on("").join(nCopies(200, "abs(")).concat("i").concat(Joiner.on("").join(nCopies(200, ")"))));
join("", nCopies(200, "abs(")).concat("i").concat(join("", nCopies(200, ")"))));
// 5000 elements cause stack overflow
ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createExpression(
Joiner.on("").join(nCopies(1000, "abs(")).concat("i").concat(Joiner.on("").join(nCopies(1000, ")")))));
join("", nCopies(1000, "abs(")).concat("i").concat(join("", nCopies(1000, ")")))));
assertThat(e.getMessage(),
startsWith("line -1:0: SQL statement is too large, causing stack overflow when generating the parsing tree: ["));
}
@ -241,11 +239,11 @@ public class SqlParserTests extends ESTestCase {
// Create expression in the form of a + a + a + ... + a
// 1000 elements is ok
new SqlParser().createExpression(Joiner.on(" + ").join(nCopies(1000, "a")));
new SqlParser().createExpression(join(" + ", nCopies(1000, "a")));
// 5000 elements cause stack overflow
ParsingException e = expectThrows(ParsingException.class, () ->
new SqlParser().createExpression(Joiner.on(" + ").join(nCopies(5000, "a"))));
new SqlParser().createExpression(join(" + ", nCopies(5000, "a"))));
assertThat(e.getMessage(),
startsWith("line -1:0: SQL statement is too large, causing stack overflow when generating the parsing tree: ["));
}
@ -255,15 +253,15 @@ public class SqlParserTests extends ESTestCase {
// 200 elements is ok
new SqlParser().createStatement(
Joiner.on(" (").join(nCopies(200, "SELECT * FROM"))
join(" (", nCopies(200, "SELECT * FROM"))
.concat("t")
.concat(Joiner.on("").join(nCopies(199, ")"))));
.concat(join("", nCopies(199, ")"))));
// 500 elements cause stack overflow
ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createStatement(
Joiner.on(" (").join(nCopies(500, "SELECT * FROM"))
join(" (", nCopies(500, "SELECT * FROM"))
.concat("t")
.concat(Joiner.on("").join(nCopies(499, ")")))));
.concat(join("", nCopies(499, ")")))));
assertThat(e.getMessage(),
startsWith("line -1:0: SQL statement is too large, causing stack overflow when generating the parsing tree: ["));
}
@ -308,4 +306,12 @@ public class SqlParserTests extends ESTestCase {
String dirStr = dir.toString();
return randomBoolean() && dirStr.equals("ASC") ? "" : " " + dirStr;
}
private String join(String delimiter, Iterable<String> strings) {
StringJoiner joiner = new StringJoiner(delimiter);
for (String s : strings) {
joiner.add(s);
}
return joiner.toString();
}
}

4
x-pack/plugin/watcher/build.gradle
View File

@ -35,8 +35,8 @@ dependencies {
// watcher deps
compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20191001.1'
compile 'com.google.guava:guava:27.1-jre' // needed by watcher for the html sanitizer
compile 'com.google.guava:failureaccess:1.0.1'
runtimeOnly 'com.google.guava:guava:27.1-jre' // needed by watcher for the html sanitizer
runtimeOnly 'com.google.guava:failureaccess:1.0.1'
compile 'com.sun.mail:jakarta.mail:1.6.4'
compile 'com.sun.activation:jakarta.activation:1.2.1'
compileOnly "org.apache.httpcomponents:httpclient:${versions.httpclient}"

3
x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java
View File

@ -5,8 +5,6 @@
*/
package org.elasticsearch.xpack.watcher.execution;
import com.google.common.collect.Iterables;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
@ -33,6 +31,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;

34
x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/HtmlSanitizer.java
View File

@ -14,12 +14,16 @@ import org.owasp.html.ElementPolicy;
import org.owasp.html.HtmlPolicyBuilder;
import org.owasp.html.PolicyFactory;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;
import java.util.function.UnaryOperator;
public class HtmlSanitizer {
@ -47,23 +51,43 @@ public class HtmlSanitizer {
private static Setting<List<String>> SETTING_SANITIZATION_DISALLOW =
Setting.listSetting("xpack.notification.email.html.sanitization.disallow", Collections.emptyList(), Function.identity(),
Property.NodeScope);
private static final MethodHandle sanitizeHandle;
static {
try {
MethodHandles.Lookup methodLookup = MethodHandles.publicLookup();
MethodType sanitizeSignature = MethodType.methodType(String.class, String.class);
sanitizeHandle = methodLookup.findVirtual(PolicyFactory.class, "sanitize", sanitizeSignature);
} catch (NoSuchMethodException|IllegalAccessException e) {
throw new RuntimeException("Missing guava on runtime classpath", e);
}
}
private final boolean enabled;
@SuppressForbidden( reason = "PolicyFactory uses guava Function")
private final PolicyFactory policy;
private final UnaryOperator<String> sanitizer;
public HtmlSanitizer(Settings settings) {
enabled = SETTING_SANITIZATION_ENABLED.get(settings);
List<String> allow = SETTING_SANITIZATION_ALLOW.get(settings);
List<String> disallow = SETTING_SANITIZATION_DISALLOW.get(settings);
policy = createCommonPolicy(allow, disallow);
// The sanitize method of PolicyFactory pulls in guava dependencies, which we want to isolate to
// runtime only rather than compile time where more guava uses can be accidentally pulled in.
// Here we lookup the sanitize method at runtime and grab a method handle to invoke.
PolicyFactory policy = createCommonPolicy(allow, disallow);
sanitizer = s -> {
try {
return (String) sanitizeHandle.invokeExact(policy, s);
} catch (Throwable e) {
throw new RuntimeException("Failed to invoke sanitize method of PolicyFactory", e);
}
};
}
public String sanitize(String html) {
if (!enabled) {
return html;
}
return policy.sanitize(html);
return sanitizer.apply(html);
}
@SuppressForbidden( reason = "PolicyFactory uses guava Function")

7
x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.watcher.notification.email.attachment;
import com.google.common.collect.ImmutableMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
@ -42,6 +41,7 @@ import java.io.InputStream;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@ -70,8 +70,9 @@ public class ReportingAttachmentParser implements EmailAttachmentParser<Reportin
private static final ObjectParser<KibanaReportingPayload, Void> PAYLOAD_PARSER =
new ObjectParser<>("reporting_attachment_kibana_payload", true, null);
static final Map<String, String> WARNINGS = ImmutableMap.of("kbn-csv-contains-formulas", "Warning: The attachment [%s] contains " +
"characters which spreadsheet applications may interpret as formulas. Please ensure that the attachment is safe prior to opening.");
static final Map<String, String> WARNINGS = Collections.singletonMap("kbn-csv-contains-formulas",
"Warning: The attachment [%s] contains characters which spreadsheet applications may interpret as formulas." +
"Please ensure that the attachment is safe prior to opening.");
static {
PARSER.declareInt(Builder::retries, ReportingAttachment.RETRIES);

9
x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionWrapperTests.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.watcher.actions;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -209,12 +208,12 @@ public class ActionWrapperTests extends ESTestCase {
List<Map<String, String>> itemsPayload = new ArrayList<>();
for (int i = 0; i < randomMaxIterations + 1; i++) {
final Action.Result actionResult = new LoggingAction.Result.Success("log_message " + i);;
final Payload singleItemPayload = new Payload.Simple(ImmutableMap.of("key", String.valueOf(i)));
itemsPayload.add(ImmutableMap.of("key", String.valueOf(i)));
final Payload singleItemPayload = new Payload.Simple(Collections.singletonMap("key", String.valueOf(i)));
itemsPayload.add(Collections.singletonMap("key", String.valueOf(i)));
when(executableAction.execute(eq("_action"), eq(ctx), eq(singleItemPayload))).thenReturn(actionResult);
}
Payload.Simple payload = new Payload.Simple(ImmutableMap.of("my_path", itemsPayload));
Payload.Simple payload = new Payload.Simple(Collections.singletonMap("my_path", itemsPayload));
when(ctx.payload()).thenReturn(payload);
when(executableAction.logger()).thenReturn(logger);
@ -233,7 +232,7 @@ public class ActionWrapperTests extends ESTestCase {
assertThat(map, hasKey("max_iterations"));
assertThat(map.get("max_iterations"), is(randomMaxIterations));
assertThat(map, hasKey("number_of_actions_executed"));
assertThat(map.get("number_of_actions_executed"), is(randomMaxIterations));
assertThat(map.get("number_of_actions_executed"), is(randomMaxIterations));
}
}

4
x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/SizeLimitInputStreamTests.java
View File

@ -12,7 +12,7 @@ import org.elasticsearch.test.ESTestCase;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import static com.google.common.base.Charsets.UTF_8;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.Matchers.is;
public class SizeLimitInputStreamTests extends ESTestCase {
@ -52,4 +52,4 @@ public class SizeLimitInputStreamTests extends ESTestCase {
}
}
}
}
}

21
x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplateTests.java
View File

@ -5,8 +5,6 @@
*/
package org.elasticsearch.xpack.watcher.notification.email;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -18,8 +16,10 @@ import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
import org.mockito.ArgumentCaptor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -185,14 +185,15 @@ public class EmailTemplateTests extends ESTestCase {
ArgumentCaptor<String> htmlSanitizeArguments = ArgumentCaptor.forClass(String.class);
//4 attachments, zero warning, one warning, two warnings, and one with html that should be stripped
Map<String, Attachment> attachments = ImmutableMap.of(
"one", new Attachment.Bytes("one", "one", randomByteArrayOfLength(100), randomAlphaOfLength(5), false, Collections.emptySet()),
"two", new Attachment.Bytes("two", "two", randomByteArrayOfLength(100), randomAlphaOfLength(5), false,
ImmutableSet.of("warning0")),
"thr", new Attachment.Bytes("thr", "thr", randomByteArrayOfLength(100), randomAlphaOfLength(5), false,
ImmutableSet.of("warning1", "warning2")),
"for", new Attachment.Bytes("for", "for", randomByteArrayOfLength(100), randomAlphaOfLength(5), false,
ImmutableSet.of("<script>warning3</script>")));
Map<String, Attachment> attachments = new HashMap<>();
attachments.put("one",
new Attachment.Bytes("one", "one", randomByteArrayOfLength(100), randomAlphaOfLength(5), false, Collections.emptySet()));
attachments.put("two", new Attachment.Bytes("two", "two", randomByteArrayOfLength(100), randomAlphaOfLength(5), false,
Collections.singleton("warning0")));
attachments.put("thr", new Attachment.Bytes("thr", "thr", randomByteArrayOfLength(100), randomAlphaOfLength(5), false,
Collections.unmodifiableSet(new HashSet<>(Arrays.asList("warning1", "warning2")))));
attachments.put("for", new Attachment.Bytes("for", "for", randomByteArrayOfLength(100), randomAlphaOfLength(5), false,
Collections.singleton("<script>warning3</script>")));
Email.Builder emailBuilder = parsedEmailTemplate.render(new MockTextTemplateEngine(), model, htmlSanitizer, attachments);
emailBuilder.id("_id");

7
x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
View File

@ -6,11 +6,11 @@
package org.elasticsearch.xpack.watcher.notification.email.attachment;
import com.fasterxml.jackson.core.io.JsonEOFException;
import com.google.common.collect.ImmutableSet;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -39,6 +39,7 @@ import java.nio.charset.StandardCharsets;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -569,8 +570,8 @@ public class ReportingAttachmentParserTests extends ESTestCase {
private ClusterService mockClusterService() {
ClusterService clusterService = mock(ClusterService.class);
ClusterSettings clusterSettings =
new ClusterSettings(Settings.EMPTY,
ImmutableSet.of(INTERVAL_SETTING, RETRIES_SETTING, REPORT_WARNING_ENABLED_SETTING, REPORT_WARNING_TEXT));
new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<Setting<?>>(
Arrays.asList(INTERVAL_SETTING, RETRIES_SETTING, REPORT_WARNING_ENABLED_SETTING, REPORT_WARNING_TEXT))));
when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
return clusterService;
}

2
x-pack/qa/security-tools-tests/build.gradle
View File

@ -4,7 +4,7 @@ dependencies {
testCompile project(xpackModule('security'))
testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
testCompile 'com.google.jimfs:jimfs:1.1'
testCompile 'com.google.guava:guava:16.0.1'
testRuntimeOnly 'com.google.guava:guava:16.0.1'
}
// add test resources from security, so certificate tool tests can use example certs

2
x-pack/snapshot-tool/build.gradle
View File

@ -32,7 +32,6 @@ dependencies {
// GCS dependencies
compile 'com.google.cloud:google-cloud-storage:1.106.0'
compile 'com.google.cloud:google-cloud-core:1.93.3'
compile 'com.google.guava:guava:26.0-jre'
compile 'com.google.http-client:google-http-client:1.34.2'
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@ -59,6 +58,7 @@ dependencies {
compile 'io.opencensus:opencensus-api:0.18.0'
compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
compile 'com.google.apis:google-api-services-storage:v1-rev20200226-1.30.9'
runtimeOnly 'com.google.guava:guava:26.0-jre'
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell