honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/rank-eval

This commit is contained in:
Christoph Büscher 2016-07-26 15:49:07 +02:00
parent 2d57fd10b1 7ed64af639
commit 91f606988a
388 changed files with 14970 additions and 8652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.gradle
View File

@ -175,6 +175,7 @@ subprojects {
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
// for transport client
"org.elasticsearch.plugin:transport-netty3-client:${version}": ':modules:transport-netty3',
"org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
"org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
"org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',

4
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -297,6 +297,10 @@ class BuildPlugin implements Plugin<Project> {
url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}"
}
}
repos.maven {
name 'netty-snapshots'
url "http://s3.amazonaws.com/download.elasticsearch.org/nettysnapshots/20160722"
}
}
/** Returns a closure which can be used with a MavenPom for removing transitive dependencies. */

100
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy
View File

@ -23,6 +23,7 @@ import org.apache.rat.anttasks.SubstringLicenseMatcher
import org.apache.rat.license.SimpleLicenseFamily
import org.elasticsearch.gradle.AntTask
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.SourceSet
@ -44,6 +45,16 @@ public class LicenseHeadersTask extends AntTask {
*/
protected List<FileCollection> javaFiles
/** Allowed license families for this project. */
@Input
List<String> approvedLicenses = ['Apache', 'Generated']
/**
* Additional license families that may be found. The key is the license category name (5 characters),
* followed by the family name and the value list of patterns to search for.
*/
protected Map<String, String> additionalLicenses = new HashMap<>()
LicenseHeadersTask() {
description = "Checks sources for missing, incorrect, or unacceptable license headers"
// Delay resolving the dependencies until after evaluation so we pick up generated sources
@ -53,6 +64,22 @@ public class LicenseHeadersTask extends AntTask {
}
}
/**
* Add a new license type.
*
* The license may be added to the {@link #approvedLicenses} using the {@code familyName}.
*
* @param categoryName A 5-character string identifier for the license
* @param familyName An expanded string name for the license
* @param pattern A pattern to search for, which if found, indicates a file contains the license
*/
public void additionalLicense(String categoryName, String familyName, String pattern) {
if (categoryName.length() != 5) {
throw new IllegalArgumentException("License category name must be exactly 5 characters, got ${categoryName}");
}
additionalLicenses.put(categoryName + familyName, pattern);
}
@Override
protected void runAnt(AntBuilder ant) {
ant.project.addTaskDefinition('ratReport', Report)
@ -64,43 +91,54 @@ public class LicenseHeadersTask extends AntTask {
// run rat, going to the file
List<FileCollection> input = javaFiles
ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
for (FileCollection dirSet : input) {
for (File dir: dirSet.srcDirs) {
// sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
if (dir.exists()) {
ant.fileset(dir: dir)
}
for (FileCollection dirSet : input) {
for (File dir: dirSet.srcDirs) {
// sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
if (dir.exists()) {
ant.fileset(dir: dir)
}
}
}
// BSD 4-clause stuff (is disallowed below)
// we keep this here, in case someone adds BSD code for some reason, it should never be allowed.
substringMatcher(licenseFamilyCategory: "BSD4 ",
licenseFamilyName: "Original BSD License (with advertising clause)") {
pattern(substring: "All advertising materials")
}
// BSD 4-clause stuff (is disallowed below)
// we keep this here, in case someone adds BSD code for some reason, it should never be allowed.
substringMatcher(licenseFamilyCategory: "BSD4 ",
licenseFamilyName: "Original BSD License (with advertising clause)") {
pattern(substring: "All advertising materials")
}
// Apache
substringMatcher(licenseFamilyCategory: "AL ",
licenseFamilyName: "Apache") {
// Apache license (ES)
pattern(substring: "Licensed to Elasticsearch under one or more contributor")
// Apache license (ASF)
pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
// this is the old-school one under some files
pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")")
}
// Apache
substringMatcher(licenseFamilyCategory: "AL ",
licenseFamilyName: "Apache") {
// Apache license (ES)
pattern(substring: "Licensed to Elasticsearch under one or more contributor")
// Apache license (ASF)
pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
// this is the old-school one under some files
pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")")
}
// Generated resources
substringMatcher(licenseFamilyCategory: "GEN ",
licenseFamilyName: "Generated") {
// parsers generated by antlr
pattern(substring: "ANTLR GENERATED CODE")
}
// Generated resources
substringMatcher(licenseFamilyCategory: "GEN ",
licenseFamilyName: "Generated") {
// parsers generated by antlr
pattern(substring: "ANTLR GENERATED CODE")
}
// approved categories
approvedLicense(familyName: "Apache")
approvedLicense(familyName: "Generated")
// license types added by the project
for (Map.Entry<String, String[]> additional : additionalLicenses.entrySet()) {
String category = additional.getKey().substring(0, 5)
String family = additional.getKey().substring(5)
substringMatcher(licenseFamilyCategory: category,
licenseFamilyName: family) {
pattern(substring: additional.getValue())
}
}
// approved categories
for (String licenseFamily : approvedLicenses) {
approvedLicense(familyName: licenseFamily)
}
}
// check the license file for any errors, this should be fast.

14
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -535,7 +535,6 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestShardsAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestThreadPoolAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]get[/\\]RestMultiGetAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]index[/\\]RestIndexAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]script[/\\]RestDeleteIndexedScriptAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]script[/\\]RestPutIndexedScriptAction.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]search[/\\]RestClearScrollAction.java" checks="LineLength" />
@ -1106,28 +1105,15 @@
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyPlugin.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptEngineService.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]BulkTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]DoubleTermsTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IPv4RangeTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndexLookupTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]LongTermsTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]MinDocCountTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]MinTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]RandomScoreFunctionTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]ScriptedMetricTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SearchFieldsTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SimpleSortTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]StringTermsTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]package-info.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovySecurityTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]mustache[/\\]MustachePlugin.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]RenderSearchTemplateTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]TemplateQueryParserTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]TemplateQueryTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]package-info.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]mustache[/\\]MustacheScriptEngineTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]mustache[/\\]MustacheTests.java" checks="LineLength" />
<suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequest.java" checks="LineLength" />
<suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequestBuilder.java" checks="LineLength" />
<suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateShardResponse.java" checks="LineLength" />

2
buildSrc/version.properties
View File

@ -13,7 +13,7 @@ jna = 4.2.2
randomizedrunner = 2.3.2
junit = 4.11
httpclient = 4.5.2
httpcore = 4.4.4
httpcore = 4.4.5
commonslogging = 1.1.3
commonscodec = 1.10
hamcrest = 1.3

35
client/benchmark/README.md Normal file
View File

@ -0,0 +1,35 @@
Steps to execute the benchmark:
1. Start Elasticsearch on the target host (ideally *not* on the same machine)
2. Create an empty index with the mapping you want to benchmark
3. Start either the RestClientBenchmark class or the TransportClientBenchmark
4. Delete the index
5. Repeat steps 2. - 4. for multiple iterations. The first iterations are intended as warmup for Elasticsearch itself. Always start the same benchmark in step 3!
4. After the benchmark: Shutdown Elasticsearch and delete the data directory
Repeat all steps above for the other benchmark candidate.
Example benchmark:
* Download benchmark data from http://benchmarks.elastic.co/corpora/geonames/documents.json.bz2 and decompress
* Use the mapping file https://github.com/elastic/rally-tracks/blob/master/geonames/mappings.json to create the index
Example command line parameter list:
```
192.168.2.2 /home/your_user_name/.rally/benchmarks/data/geonames/documents.json geonames type 8647880 5000 "{ \"query\": { \"match_phrase\": { \"name\": \"Sankt Georgen\" } } }\""
```
The parameters are in order:
* Benchmark target host IP (the host where Elasticsearch is running)
* full path to the file that should be bulk indexed
* name of the index
* name of the (sole) type in the index
* number of documents in the file
* bulk size
* a search request body (remember to escape double quotes). The `TransportClientBenchmark` uses `QueryBuilders.wrapperQuery()` internally which automatically adds a root key `query`, so it must not be present in the command line parameter.
You should also define a few GC-related settings `-Xms4096M -Xmx4096M -XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails` and keep an eye on GC activity. You can also define `-XX:+PrintCompilation` to see JIT activity.

48
client/benchmark/build.gradle Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.build'
group = 'org.elasticsearch.client'
// never try to invoke tests on the benchmark project - there aren't any
check.dependsOn.remove(test)
// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
task test(type: Test, overwrite: true)
dependencies {
compile 'org.apache.commons:commons-math3:3.2'
compile("org.elasticsearch.client:rest:${version}")
// for transport client
compile("org.elasticsearch:elasticsearch:${version}")
compile("org.elasticsearch.client:transport:${version}")
compile project(path: ':modules:transport-netty3', configuration: 'runtime')
compile project(path: ':modules:transport-netty4', configuration: 'runtime')
compile project(path: ':modules:reindex', configuration: 'runtime')
compile project(path: ':modules:lang-mustache', configuration: 'runtime')
compile project(path: ':modules:percolator', configuration: 'runtime')
}
// No licenses for our benchmark deps (we don't ship benchmarks)
dependencyLicenses.enabled = false
extraArchive {
javadoc = false
}

88
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark;
import org.elasticsearch.client.benchmark.ops.bulk.BulkBenchmarkTask;
import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
import org.elasticsearch.client.benchmark.ops.search.SearchBenchmarkTask;
import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;
import org.elasticsearch.common.SuppressForbidden;
import java.io.Closeable;
public abstract class AbstractBenchmark<T extends Closeable> {
private static final int SEARCH_BENCHMARK_ITERATIONS = 10_000;
protected abstract T client(String benchmarkTargetHost) throws Exception;
protected abstract BulkRequestExecutor bulkRequestExecutor(T client, String indexName, String typeName);
protected abstract SearchRequestExecutor searchRequestExecutor(T client, String indexName);
@SuppressForbidden(reason = "system out is ok for a command line tool")
public final void run(String[] args) throws Exception {
if (args.length < 6) {
System.err.println(
"usage: benchmarkTargetHostIp indexFilePath indexName typeName numberOfDocuments bulkSize [search request body]");
System.exit(1);
}
String benchmarkTargetHost = args[0];
String indexFilePath = args[1];
String indexName = args[2];
String typeName = args[3];
int totalDocs = Integer.valueOf(args[4]);
int bulkSize = Integer.valueOf(args[5]);
int totalIterationCount = (int) Math.floor(totalDocs / bulkSize);
// consider 40% of all iterations as warmup iterations
int warmupIterations = (int) (0.4d * totalIterationCount);
int iterations = totalIterationCount - warmupIterations;
String searchBody = (args.length == 7) ? args[6] : null;
T client = client(benchmarkTargetHost);
BenchmarkRunner benchmark = new BenchmarkRunner(warmupIterations, iterations,
new BulkBenchmarkTask(
bulkRequestExecutor(client, indexName, typeName), indexFilePath, warmupIterations + iterations, bulkSize));
try {
benchmark.run();
if (searchBody != null) {
for (int run = 1; run <= 5; run++) {
System.out.println("=============");
System.out.println(" Trial run " + run);
System.out.println("=============");
for (int throughput = 100; throughput <= 100_000; throughput *= 10) {
//request a GC between trials to reduce the likelihood of a GC occurring in the middle of a trial.
System.gc();
BenchmarkRunner searchBenchmark = new BenchmarkRunner(SEARCH_BENCHMARK_ITERATIONS, SEARCH_BENCHMARK_ITERATIONS,
new SearchBenchmarkTask(
searchRequestExecutor(client, indexName), searchBody, 2 * SEARCH_BENCHMARK_ITERATIONS, throughput));
System.out.printf("Target throughput = %d ops / s%n", throughput);
searchBenchmark.run();
}
}
}
} finally {
client.close();
}
}
}

84
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark;
import org.elasticsearch.client.benchmark.metrics.Metrics;
import org.elasticsearch.client.benchmark.metrics.MetricsCalculator;
import org.elasticsearch.client.benchmark.metrics.Sample;
import org.elasticsearch.client.benchmark.metrics.SampleRecorder;
import org.elasticsearch.common.SuppressForbidden;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
public final class BenchmarkRunner {
private final int warmupIterations;
private final int iterations;
private final BenchmarkTask task;
public BenchmarkRunner(int warmupIterations, int iterations, BenchmarkTask task) {
this.warmupIterations = warmupIterations;
this.iterations = iterations;
this.task = task;
}
@SuppressForbidden(reason = "system out is ok for a command line tool")
public void run() throws Exception {
SampleRecorder recorder = new SampleRecorder(warmupIterations, iterations);
System.out.printf("Running %s with %d warmup iterations and %d iterations.%n",
task.getClass().getSimpleName(), warmupIterations, iterations);
try {
task.setUp(recorder);
task.run();
task.tearDown();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
return;
}
List<Sample> samples = recorder.getSamples();
final List<Metrics> summaryMetrics = MetricsCalculator.calculate(samples);
if (summaryMetrics.isEmpty()) {
System.out.println("No results.");
}
for (Metrics metrics : summaryMetrics) {
System.out.printf(Locale.ROOT, "Operation: %s%n", metrics.operation);
String stats = String.format(Locale.ROOT,
"Throughput = %f ops/s, p90 = %f ms, p95 = %f ms, p99 = %f ms, p99.9 = %f ms, p99.99 = %f ms",
metrics.throughput,
metrics.serviceTimeP90, metrics.serviceTimeP95,
metrics.serviceTimeP99, metrics.serviceTimeP999,
metrics.serviceTimeP9999);
System.out.println(repeat(stats.length(), '-'));
System.out.println(stats);
System.out.printf("success count = %d, error count = %d%n", metrics.successCount, metrics.errorCount);
System.out.println(repeat(stats.length(), '-'));
}
}
private String repeat(int times, char character) {
char[] characters = new char[times];
Arrays.fill(characters, character);
return new String(characters);
}
}

29
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark;
import org.elasticsearch.client.benchmark.metrics.SampleRecorder;
public interface BenchmarkTask {
void setUp(SampleRecorder sampleRecorder) throws Exception;
void run() throws Exception;
void tearDown() throws Exception;
}

45
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.metrics;
public final class Metrics {
public final String operation;
public final long successCount;
public final long errorCount;
public final double throughput;
public final double serviceTimeP90;
public final double serviceTimeP95;
public final double serviceTimeP99;
public final double serviceTimeP999;
public final double serviceTimeP9999;
public Metrics(String operation, long successCount, long errorCount, double throughput,
double serviceTimeP90, double serviceTimeP95, double serviceTimeP99,
double serviceTimeP999, double serviceTimeP9999) {
this.operation = operation;
this.successCount = successCount;
this.errorCount = errorCount;
this.throughput = throughput;
this.serviceTimeP90 = serviceTimeP90;
this.serviceTimeP95 = serviceTimeP95;
this.serviceTimeP99 = serviceTimeP99;
this.serviceTimeP999 = serviceTimeP999;
this.serviceTimeP9999 = serviceTimeP9999;
}
}

80
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.metrics;
import org.apache.commons.math3.stat.StatUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public final class MetricsCalculator {
public static List<Metrics> calculate(Collection<Sample> samples) {
Map<String, List<Sample>> samplesPerOperation = groupByOperation(samples);
return calculateMetricsPerOperation(samplesPerOperation);
}
private static Map<String, List<Sample>> groupByOperation(Collection<Sample> samples) {
Map<String, List<Sample>> samplesPerOperation = new HashMap<>();
for (Sample sample : samples) {
if (!samplesPerOperation.containsKey(sample.getOperation())) {
samplesPerOperation.put(sample.getOperation(), new ArrayList<>());
}
samplesPerOperation.get(sample.getOperation()).add(sample);
}
return samplesPerOperation;
}
private static List<Metrics> calculateMetricsPerOperation(Map<String, List<Sample>> samplesPerOperation) {
List<Metrics> metrics = new ArrayList<>();
for (Map.Entry<String, List<Sample>> operationAndMetrics : samplesPerOperation.entrySet()) {
List<Sample> samples = operationAndMetrics.getValue();
double[] serviceTimes = new double[samples.size()];
int it = 0;
long firstStart = Long.MAX_VALUE;
long latestEnd = Long.MIN_VALUE;
for (Sample sample : samples) {
firstStart = Math.min(sample.getStartTimestamp(), firstStart);
latestEnd = Math.max(sample.getStopTimestamp(), latestEnd);
serviceTimes[it++] = sample.getServiceTime();
}
metrics.add(new Metrics(operationAndMetrics.getKey(),
samples.stream().filter((r) -> r.isSuccess()).count(),
samples.stream().filter((r) -> !r.isSuccess()).count(),
// throughput calculation is based on the total (Wall clock) time it took to generate all samples
calculateThroughput(samples.size(), latestEnd - firstStart),
// convert ns -> ms without losing precision
StatUtils.percentile(serviceTimes, 90.0d) / TimeUnit.MILLISECONDS.toNanos(1L),
StatUtils.percentile(serviceTimes, 95.0d) / TimeUnit.MILLISECONDS.toNanos(1L),
StatUtils.percentile(serviceTimes, 99.0d) / TimeUnit.MILLISECONDS.toNanos(1L),
StatUtils.percentile(serviceTimes, 99.9d) / TimeUnit.MILLISECONDS.toNanos(1L),
StatUtils.percentile(serviceTimes, 99.99d) / TimeUnit.MILLISECONDS.toNanos(1L)));
}
return metrics;
}
private static double calculateThroughput(int sampleSize, double duration) {
return sampleSize * (TimeUnit.SECONDS.toNanos(1L) / duration);
}
}

54
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.metrics;
public final class Sample {
private final String operation;
private final long startTimestamp;
private final long stopTimestamp;
private final boolean success;
public Sample(String operation, long startTimestamp, long stopTimestamp, boolean success) {
this.operation = operation;
this.startTimestamp = startTimestamp;
this.stopTimestamp = stopTimestamp;
this.success = success;
}
public String getOperation() {
return operation;
}
public boolean isSuccess() {
return success;
}
public long getStartTimestamp() {
return startTimestamp;
}
public long getStopTimestamp() {
return stopTimestamp;
}
public long getServiceTime() {
// this is *not* latency, we're not including wait time in the queue (on purpose)
return stopTimestamp - startTimestamp;
}
}

51
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.metrics;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Stores measurement samples.
*
* This class is NOT threadsafe.
*/
public final class SampleRecorder {
private final int warmupIterations;
private final List<Sample> samples;
private int currentIteration;
public SampleRecorder(int warmupIterations, int iterations) {
this.warmupIterations = warmupIterations;
this.samples = new ArrayList<>(iterations);
}
public void addSample(Sample sample) {
currentIteration++;
// only add samples after warmup
if (currentIteration > warmupIterations) {
samples.add(sample);
}
}
public List<Sample> getSamples() {
return Collections.unmodifiableList(samples);
}
}

174
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.ops.bulk;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.client.benchmark.BenchmarkTask;
import org.elasticsearch.client.benchmark.metrics.Sample;
import org.elasticsearch.client.benchmark.metrics.SampleRecorder;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class BulkBenchmarkTask implements BenchmarkTask {
private final BulkRequestExecutor requestExecutor;
private final String indexFilePath;
private final int totalIterations;
private final int bulkSize;
private LoadGenerator generator;
private ExecutorService executorService;
public BulkBenchmarkTask(BulkRequestExecutor requestExecutor, String indexFilePath, int totalIterations, int bulkSize) {
this.requestExecutor = requestExecutor;
this.indexFilePath = indexFilePath;
this.totalIterations = totalIterations;
this.bulkSize = bulkSize;
}
@Override
@SuppressForbidden(reason = "PathUtils#get is fine - we don't have environment here")
public void setUp(SampleRecorder sampleRecorder) {
BlockingQueue<List<String>> bulkQueue = new ArrayBlockingQueue<>(256);
BulkIndexer runner = new BulkIndexer(bulkQueue, totalIterations, sampleRecorder, requestExecutor);
executorService = Executors.newSingleThreadExecutor((r) -> new Thread(r, "bulk-index-runner"));
executorService.submit(runner);
generator = new LoadGenerator(PathUtils.get(indexFilePath), bulkQueue, bulkSize);
}
@Override
@SuppressForbidden(reason = "system out is ok for a command line tool")
public void run() throws Exception {
generator.execute();
// when the generator is done, there are no more data -> shutdown client
executorService.shutdown();
//We need to wait until the queue is drained
final boolean finishedNormally = executorService.awaitTermination(20, TimeUnit.MINUTES);
if (finishedNormally == false) {
System.err.println("Background tasks are still running after timeout on enclosing pool. Forcing pool shutdown.");
executorService.shutdownNow();
}
}
@Override
public void tearDown() {
//no op
}
private static final class LoadGenerator {
private final Path bulkDataFile;
private final BlockingQueue<List<String>> bulkQueue;
private final int bulkSize;
public LoadGenerator(Path bulkDataFile, BlockingQueue<List<String>> bulkQueue, int bulkSize) {
this.bulkDataFile = bulkDataFile;
this.bulkQueue = bulkQueue;
this.bulkSize = bulkSize;
}
@SuppressForbidden(reason = "Classic I/O is fine in non-production code")
public void execute() {
try (BufferedReader reader = Files.newBufferedReader(bulkDataFile, StandardCharsets.UTF_8)) {
String line;
int bulkIndex = 0;
List<String> bulkData = new ArrayList<>(bulkSize);
while ((line = reader.readLine()) != null) {
if (bulkIndex == bulkSize) {
sendBulk(bulkData);
// reset data structures
bulkData = new ArrayList<>(bulkSize);
bulkIndex = 0;
}
bulkData.add(line);
bulkIndex++;
}
// also send the last bulk:
if (bulkIndex > 0) {
sendBulk(bulkData);
}
} catch (IOException e) {
throw new ElasticsearchException(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private void sendBulk(List<String> bulkData) throws InterruptedException {
bulkQueue.put(bulkData);
}
}
private static final class BulkIndexer implements Runnable {
private static final ESLogger logger = ESLoggerFactory.getLogger(BulkIndexer.class.getName());
private final BlockingQueue<List<String>> bulkData;
private final int totalIterations;
private final BulkRequestExecutor bulkRequestExecutor;
private final SampleRecorder sampleRecorder;
public BulkIndexer(BlockingQueue<List<String>> bulkData, int totalIterations, SampleRecorder sampleRecorder,
BulkRequestExecutor bulkRequestExecutor) {
this.bulkData = bulkData;
this.totalIterations = totalIterations;
this.bulkRequestExecutor = bulkRequestExecutor;
this.sampleRecorder = sampleRecorder;
}
@Override
public void run() {
for (int iteration = 0; iteration < totalIterations; iteration++) {
boolean success = false;
List<String> currentBulk;
try {
currentBulk = bulkData.take();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
// Yes, this approach is prone to coordinated omission *but* we have to consider that we want to benchmark a closed system
// with backpressure here instead of an open system. So this is actually correct in this case.
long start = System.nanoTime();
try {
success = bulkRequestExecutor.bulkIndex(currentBulk);
} catch (Exception ex) {
logger.warn("Error while executing bulk request", ex);
}
long stop = System.nanoTime();
sampleRecorder.addSample(new Sample("bulk", start, stop, success));
}
}
}
}

25
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.ops.bulk;
import java.util.List;
public interface BulkRequestExecutor {
boolean bulkIndex(List<String> bulkData);
}

86
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.ops.search;
import org.elasticsearch.client.benchmark.BenchmarkTask;
import org.elasticsearch.client.benchmark.metrics.Sample;
import org.elasticsearch.client.benchmark.metrics.SampleRecorder;
import java.util.concurrent.TimeUnit;
public class SearchBenchmarkTask implements BenchmarkTask {
private static final long MICROS_PER_SEC = TimeUnit.SECONDS.toMicros(1L);
private static final long NANOS_PER_MICRO = TimeUnit.MICROSECONDS.toNanos(1L);
private final SearchRequestExecutor searchRequestExecutor;
private final String searchRequestBody;
private final int iterations;
private final int targetThroughput;
private SampleRecorder sampleRecorder;
public SearchBenchmarkTask(SearchRequestExecutor searchRequestExecutor, String body, int iterations, int targetThroughput) {
this.searchRequestExecutor = searchRequestExecutor;
this.searchRequestBody = body;
this.iterations = iterations;
this.targetThroughput = targetThroughput;
}
@Override
public void setUp(SampleRecorder sampleRecorder) throws Exception {
this.sampleRecorder = sampleRecorder;
}
@Override
public void run() throws Exception {
for (int iteration = 0; iteration < this.iterations; iteration++) {
final long start = System.nanoTime();
boolean success = searchRequestExecutor.search(searchRequestBody);
final long stop = System.nanoTime();
sampleRecorder.addSample(new Sample("search", start, stop, success));
int waitTime = (int) Math.floor(MICROS_PER_SEC / targetThroughput - (stop - start) / NANOS_PER_MICRO);
if (waitTime > 0) {
// Thread.sleep() time is not very accurate (it's most of the time around 1 - 2 ms off)
// so we rather busy spin for the last few microseconds. Still not entirely accurate but way closer
waitMicros(waitTime);
}
}
}
private void waitMicros(int waitTime) throws InterruptedException {
int millis = waitTime / 1000;
int micros = waitTime % 1000;
if (millis > 0) {
Thread.sleep(millis);
}
// busy spin for the rest of the time
if (micros > 0) {
long end = System.nanoTime() + 1000L * micros;
while (end > System.nanoTime()) {
// busy spin
}
}
}
@Override
public void tearDown() throws Exception {
// no op
}
}

23
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.ops.search;
public interface SearchRequestExecutor {
boolean search(String source);
}

108
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.rest;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.benchmark.AbstractBenchmark;
import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public final class RestClientBenchmark extends AbstractBenchmark<RestClient> {
public static void main(String[] args) throws Exception {
RestClientBenchmark b = new RestClientBenchmark();
b.run(args);
}
@Override
protected RestClient client(String benchmarkTargetHost) {
return RestClient.builder(new HttpHost(benchmarkTargetHost, 9200)).build();
}
@Override
protected BulkRequestExecutor bulkRequestExecutor(RestClient client, String indexName, String typeName) {
return new RestBulkRequestExecutor(client, indexName, typeName);
}
@Override
protected SearchRequestExecutor searchRequestExecutor(RestClient client, String indexName) {
return new RestSearchRequestExecutor(client, indexName);
}
private static final class RestBulkRequestExecutor implements BulkRequestExecutor {
private final RestClient client;
private final String actionMetaData;
public RestBulkRequestExecutor(RestClient client, String index, String type) {
this.client = client;
this.actionMetaData = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
}
@Override
public boolean bulkIndex(List<String> bulkData) {
StringBuilder bulkRequestBody = new StringBuilder();
for (String bulkItem : bulkData) {
bulkRequestBody.append(actionMetaData);
bulkRequestBody.append(bulkItem);
bulkRequestBody.append("\n");
}
HttpEntity entity = new NStringEntity(bulkRequestBody.toString(), ContentType.APPLICATION_JSON);
try {
Response response = client.performRequest("POST", "/geonames/type/_bulk", Collections.emptyMap(), entity);
return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
} catch (Exception e) {
throw new ElasticsearchException(e);
}
}
}
private static final class RestSearchRequestExecutor implements SearchRequestExecutor {
private final RestClient client;
private final String endpoint;
private RestSearchRequestExecutor(RestClient client, String indexName) {
this.client = client;
this.endpoint = "/" + indexName + "/_search";
}
@Override
public boolean search(String source) {
HttpEntity searchBody = new NStringEntity(source, StandardCharsets.UTF_8);
try {
Response response = client.performRequest("GET", endpoint, Collections.emptyMap(), searchBody);
return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
} catch (IOException e) {
throw new ElasticsearchException(e);
}
}
}
}

117
client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.benchmark.transport;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.benchmark.AbstractBenchmark;
import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.ExecutionException;
public final class TransportClientBenchmark extends AbstractBenchmark<TransportClient> {
public static void main(String[] args) throws Exception {
TransportClientBenchmark benchmark = new TransportClientBenchmark();
benchmark.run(args);
}
@Override
protected TransportClient client(String benchmarkTargetHost) throws Exception {
TransportClient client = new PreBuiltTransportClient(Settings.EMPTY);
client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(benchmarkTargetHost), 9300));
return client;
}
@Override
protected BulkRequestExecutor bulkRequestExecutor(TransportClient client, String indexName, String typeName) {
return new TransportBulkRequestExecutor(client, indexName, typeName);
}
@Override
protected SearchRequestExecutor searchRequestExecutor(TransportClient client, String indexName) {
return new TransportSearchRequestExecutor(client, indexName);
}
private static final class TransportBulkRequestExecutor implements BulkRequestExecutor {
private final TransportClient client;
private final String indexName;
private final String typeName;
public TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) {
this.client = client;
this.indexName = indexName;
this.typeName = typeName;
}
@Override
public boolean bulkIndex(List<String> bulkData) {
BulkRequestBuilder builder = client.prepareBulk();
for (String bulkItem : bulkData) {
builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8)));
}
BulkResponse bulkResponse;
try {
bulkResponse = builder.execute().get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (ExecutionException e) {
throw new ElasticsearchException(e);
}
return !bulkResponse.hasFailures();
}
}
private static final class TransportSearchRequestExecutor implements SearchRequestExecutor {
private final TransportClient client;
private final String indexName;
private TransportSearchRequestExecutor(TransportClient client, String indexName) {
this.client = client;
this.indexName = indexName;
}
@Override
public boolean search(String source) {
final SearchResponse response;
try {
response = client.prepareSearch(indexName).setQuery(QueryBuilders.wrapperQuery(source)).execute().get();
return response.status() == RestStatus.OK;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (ExecutionException e) {
throw new ElasticsearchException(e);
}
}
}
}

9
client/benchmark/src/main/resources/log4j.properties Normal file
View File

@ -0,0 +1,9 @@
es.logger.level=INFO
log4j.rootLogger=${es.logger.level}, out
log4j.logger.org.apache.http=INFO, out
log4j.additivity.org.apache.http=false
log4j.appender.out=org.apache.log4j.ConsoleAppender
log4j.appender.out.layout=org.apache.log4j.PatternLayout
log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n

7
client/rest/build.gradle
View File

@ -31,6 +31,8 @@ group = 'org.elasticsearch.client'
dependencies {
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
compile "org.apache.httpcomponents:httpasyncclient:4.1.2"
compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
compile "commons-codec:commons-codec:${versions.commonscodec}"
compile "commons-logging:commons-logging:${versions.commonslogging}"
@ -56,6 +58,11 @@ forbiddenApisTest {
signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
}
dependencyLicenses {
mapping from: /http.*/, to: 'httpclient'
mapping from: /commons-.*/, to: 'commons'
}
//JarHell is part of es core, which we don't want to pull in
jarHell.enabled=false

0
client/rest/licenses/commons-codec-LICENSE.txt → client/rest/licenses/commons-LICENSE.txt
View File

0
client/rest/licenses/commons-codec-NOTICE.txt → client/rest/licenses/commons-NOTICE.txt
View File

6
client/rest/licenses/commons-logging-NOTICE.txt
View File

@ -1,6 +0,0 @@
Apache Commons Logging
Copyright 2003-2014 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

1
client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 Normal file
View File

@ -0,0 +1 @@
95aa3e6fb520191a0970a73cf09f62948ee614be

1
client/rest/licenses/httpcore-4.4.4.jar.sha1
View File

@ -1 +0,0 @@
b31526a230871fbe285fbcbe2813f9c0839ae9b0

1
client/rest/licenses/httpcore-4.4.5.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e7501a1b34325abb00d17dde96150604a0658b54

558
client/rest/licenses/httpcore-LICENSE.txt
View File

@ -1,558 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
=========================================================================
This project includes Public Suffix List copied from
<https://publicsuffix.org/list/effective_tld_names.dat>
licensed under the terms of the Mozilla Public License, v. 2.0
Full license text: <http://mozilla.org/MPL/2.0/>
Mozilla Public License Version 2.0
==================================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to
the creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used
by a Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached
the notice in Exhibit A, the Executable Form of such Source Code
Form, and Modifications of such Source Code Form, in each case
including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
(a) that the initial Contributor has attached the notice described
in Exhibit B to the Covered Software; or
(b) that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the
terms of a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible,
whether at the time of the initial grant or subsequently, any and
all of the rights conveyed by this License.
1.10. "Modifications"
means any of the following:
(a) any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered
Software; or
(b) any new file in Source Code Form that contains any Covered
Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the
License, by the making, using, selling, offering for sale, having
made, import, or transfer of either its Contributions or its
Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU
Lesser General Public License, Version 2.1, the GNU Affero General
Public License, Version 3.0, or any later versions of those
licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that
controls, is controlled by, or is under common control with You. For
purposes of this definition, "control" means (a) the power, direct
or indirect, to cause the direction or management of such entity,
whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial
ownership of such entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
(b) under Patent Claims of such Contributor to make, use, sell, offer
for sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
(a) for any code that a Contributor has removed from Covered Software;
or
(b) for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
(c) under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights
to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
in Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
(a) such Covered Software must also be made available in Source Code
Form, as described in Section 3.1, and You must inform recipients of
the Executable Form how they can obtain a copy of such Source Code
Form by reasonable means in a timely manner, at a charge no more
than the cost of distribution to the recipient; and
(b) You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter
the recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty,
or limitations of liability) contained within the Source Code Form of
the Covered Software, except that You may alter any license notices to
the extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Software due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description must
be placed in a text file included with all distributions of the Covered
Software under this License. Except to the extent prohibited by statute
or regulation, such description must be sufficiently detailed for a
recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically
if You fail to comply with any of its terms. However, if You become
compliant, then the rights granted under this License from a particular
Contributor are reinstated (a) provisionally, unless and until such
Contributor explicitly and finally terminates Your grants, and (b) on an
ongoing basis, if such Contributor fails to notify You of the
non-compliance by some reasonable means prior to 60 days after You have
come back into compliance. Moreover, Your grants from a particular
Contributor are reinstated on an ongoing basis if such Contributor
notifies You of the non-compliance by some reasonable means, this is the
first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after
Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
end user license agreements (excluding distributors and resellers) which
have been validly granted by You or Your distributors under this License
prior to termination shall survive termination.
************************************************************************
* *
* 6. Disclaimer of Warranty *
* ------------------------- *
* *
* Covered Software is provided under this License on an "as is" *
* basis, without warranty of any kind, either expressed, implied, or *
* statutory, including, without limitation, warranties that the *
* Covered Software is free of defects, merchantable, fit for a *
* particular purpose or non-infringing. The entire risk as to the *
* quality and performance of the Covered Software is with You. *
* Should any Covered Software prove defective in any respect, You *
* (not any Contributor) assume the cost of any necessary servicing, *
* repair, or correction. This disclaimer of warranty constitutes an *
* essential part of this License. No use of any Covered Software is *
* authorized under this License except under this disclaimer. *
* *
************************************************************************
************************************************************************
* *
* 7. Limitation of Liability *
* -------------------------- *
* *
* Under no circumstances and under no legal theory, whether tort *
* (including negligence), contract, or otherwise, shall any *
* Contributor, or anyone who distributes Covered Software as *
* permitted above, be liable to You for any direct, indirect, *
* special, incidental, or consequential damages of any character *
* including, without limitation, damages for lost profits, loss of *
* goodwill, work stoppage, computer failure or malfunction, or any *
* and all other commercial damages or losses, even if such party *
* shall have been informed of the possibility of such damages. This *
* limitation of liability shall not apply to liability for death or *
* personal injury resulting from such party's negligence to the *
* extent applicable law prohibits such limitation. Some *
* jurisdictions do not allow the exclusion or limitation of *
* incidental or consequential damages, so this exclusion and *
* limitation may not apply to You. *
* *
************************************************************************
8. Litigation
-------------
Any litigation relating to this License may be brought only in the
courts of a jurisdiction where the defendant maintains its principal
place of business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions.
Nothing in this Section shall prevent a party's ability to bring
cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides
that the language of a contract shall be construed against the drafter
shall not be used to construe this License against a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the
notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to look
for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.

6
client/rest/licenses/httpcore-NOTICE.txt
View File

@ -1,6 +0,0 @@
Apache HttpComponents Client
Copyright 1999-2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

1
client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 Normal file
View File

@ -0,0 +1 @@
f4be009e7505f6ceddf21e7960c759f413f15056

111
client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.ContentTooLongException;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.ContentDecoder;
import org.apache.http.nio.IOControl;
import org.apache.http.nio.entity.ContentBufferEntity;
import org.apache.http.nio.protocol.AbstractAsyncResponseConsumer;
import org.apache.http.nio.util.ByteBufferAllocator;
import org.apache.http.nio.util.HeapByteBufferAllocator;
import org.apache.http.nio.util.SimpleInputBuffer;
import org.apache.http.protocol.HttpContext;
import java.io.IOException;
/**
* Default implementation of {@link org.apache.http.nio.protocol.HttpAsyncResponseConsumer}. Buffers the whole
* response content in heap memory, meaning that the size of the buffer is equal to the content-length of the response.
* Limits the size of responses that can be read to {@link #DEFAULT_BUFFER_LIMIT} by default, configurable value.
* Throws an exception in case the entity is longer than the configured buffer limit.
*/
public class HeapBufferedAsyncResponseConsumer extends AbstractAsyncResponseConsumer<HttpResponse> {
//default buffer limit is 10MB
public static final int DEFAULT_BUFFER_LIMIT = 10 * 1024 * 1024;
private final int bufferLimit;
private volatile HttpResponse response;
private volatile SimpleInputBuffer buf;
/**
* Creates a new instance of this consumer with a buffer limit of {@link #DEFAULT_BUFFER_LIMIT}
*/
public HeapBufferedAsyncResponseConsumer() {
this.bufferLimit = DEFAULT_BUFFER_LIMIT;
}
/**
* Creates a new instance of this consumer with the provided buffer limit
*/
public HeapBufferedAsyncResponseConsumer(int bufferLimit) {
if (bufferLimit <= 0) {
throw new IllegalArgumentException("bufferLimit must be greater than 0");
}
this.bufferLimit = bufferLimit;
}
@Override
protected void onResponseReceived(HttpResponse response) throws HttpException, IOException {
this.response = response;
}
@Override
protected void onEntityEnclosed(HttpEntity entity, ContentType contentType) throws IOException {
long len = entity.getContentLength();
if (len > bufferLimit) {
throw new ContentTooLongException("entity content is too long [" + len +
"] for the configured buffer limit [" + bufferLimit + "]");
}
if (len < 0) {
len = 4096;
}
this.buf = new SimpleInputBuffer((int) len, getByteBufferAllocator());
this.response.setEntity(new ContentBufferEntity(entity, this.buf));
}
/**
* Returns the instance of {@link ByteBufferAllocator} to use for content buffering.
* Allows to plug in any {@link ByteBufferAllocator} implementation.
*/
protected ByteBufferAllocator getByteBufferAllocator() {
return HeapByteBufferAllocator.INSTANCE;
}
@Override
protected void onContentReceived(ContentDecoder decoder, IOControl ioctrl) throws IOException {
this.buf.consumeContent(decoder);
}
@Override
protected HttpResponse buildResult(HttpContext context) throws Exception {
return response;
}
@Override
protected void releaseResources() {
response = null;
}
}

2
client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
View File

@ -81,7 +81,7 @@ final class RequestLogger {
/**
* Logs a request that failed
*/
static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, IOException e) {
static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + "] failed", e);
}

18
client/rest/src/main/java/org/elasticsearch/client/Response.java
View File

@ -22,26 +22,23 @@ package org.elasticsearch.client;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.RequestLine;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import java.io.Closeable;
import java.io.IOException;
import java.util.Objects;
/**
* Holds an elasticsearch response. It wraps the {@link CloseableHttpResponse} response and associates it with
* Holds an elasticsearch response. It wraps the {@link HttpResponse} returned and associates it with
* its corresponding {@link RequestLine} and {@link HttpHost}.
* It must be closed to free any resource held by it, as well as the corresponding connection in the connection pool.
*/
public class Response implements Closeable {
public final class Response {
private final RequestLine requestLine;
private final HttpHost host;
private final CloseableHttpResponse response;
private final HttpResponse response;
Response(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
Response(RequestLine requestLine, HttpHost host, HttpResponse response) {
Objects.requireNonNull(requestLine, "requestLine cannot be null");
Objects.requireNonNull(host, "node cannot be null");
Objects.requireNonNull(response, "response cannot be null");
@ -107,9 +104,4 @@ public class Response implements Closeable {
", response=" + response.getStatusLine() +
'}';
}
@Override
public void close() throws IOException {
this.response.close();
}
}

30
client/rest/src/main/java/org/elasticsearch/client/ResponseException.java
View File

@ -23,44 +23,26 @@ import java.io.IOException;
/**
* Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error.
* Note that the response body gets passed in as a string and read eagerly, which means that the Response object
* is expected to be closed and available only to read metadata like status line, request line, response headers.
* Holds the response that was returned.
*/
public class ResponseException extends IOException {
public final class ResponseException extends IOException {
private Response response;
private final String responseBody;
ResponseException(Response response, String responseBody) throws IOException {
super(buildMessage(response,responseBody));
ResponseException(Response response) throws IOException {
super(buildMessage(response));
this.response = response;
this.responseBody = responseBody;
}
private static String buildMessage(Response response, String responseBody) {
String message = response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri()
private static String buildMessage(Response response) {
return response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri()
+ ": " + response.getStatusLine().toString();
if (responseBody != null) {
message += "\n" + responseBody;
}
return message;
}
/**
* Returns the {@link Response} that caused this exception to be thrown.
* Expected to be used only to read metadata like status line, request line, response headers. The response body should
* be retrieved using {@link #getResponseBody()}
*/
public Response getResponse() {
return response;
}
/**
* Returns the response body as a string or null if there wasn't any.
* The body is eagerly consumed when an ResponseException gets created, and its corresponding Response
* gets closed straightaway so this method is the only way to get back the response body that was returned.
*/
public String getResponseBody() {
return responseBody;
}
}

40
client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
/**
* Listener to be provided when calling async performRequest methods provided by {@link RestClient}.
* Those methods that do accept a listener will return immediately, execute asynchronously, and notify
* the listener whenever the request yielded a response, or failed with an exception.
*/
public interface ResponseListener {
/**
* Method invoked if the request yielded a successful response
*/
void onSuccess(Response response);
/**
* Method invoked if the request failed. There are two main categories of failures: connection failures (usually
* {@link java.io.IOException}s, or responses that were treated as errors based on their error response code
* ({@link ResponseException}s).
*/
void onFailure(Exception exception);
}

580
client/rest/src/main/java/org/elasticsearch/client/RestClient.java
View File

@ -20,14 +20,12 @@ package org.elasticsearch.client;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpOptions;
@ -37,20 +35,22 @@ import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpTrace;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.nio.client.methods.HttpAsyncMethods;
import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
import java.io.Closeable;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -58,30 +58,31 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
* Client that connects to an elasticsearch cluster through http.
* Must be created using {@link Builder}, which allows to set all the different options or just rely on defaults.
* Must be created using {@link RestClientBuilder}, which allows to set all the different options or just rely on defaults.
* The hosts that are part of the cluster need to be provided at creation time, but can also be replaced later
* by calling {@link #setHosts(HttpHost...)}.
* The method {@link #performRequest(String, String, Map, HttpEntity, Header...)} allows to send a request to the cluster. When
* sending a request, a host gets selected out of the provided ones in a round-robin fashion. Failing hosts are marked dead and
* retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously
* failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead nodes that
* deserve a retry) are retried till one responds or none of them does, in which case an {@link IOException} will be thrown.
* deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown.
*
* Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format.
*/
public final class RestClient implements Closeable {
private static final Log logger = LogFactory.getLog(RestClient.class);
public static ContentType JSON_CONTENT_TYPE = ContentType.create("application/json", Consts.UTF_8);
private final CloseableHttpClient client;
//we don't rely on default headers supported by HttpClient as those cannot be replaced, plus it would get hairy
//when we create the HttpClient instance on our own as there would be two different ways to set the default headers.
private final CloseableHttpAsyncClient client;
//we don't rely on default headers supported by HttpAsyncClient as those cannot be replaced
private final Header[] defaultHeaders;
private final long maxRetryTimeoutMillis;
private final AtomicInteger lastHostIndex = new AtomicInteger(0);
@ -89,7 +90,7 @@ public final class RestClient implements Closeable {
private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
private final FailureListener failureListener;
RestClient(CloseableHttpClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders,
RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders,
HttpHost[] hosts, FailureListener failureListener) {
this.client = client;
this.maxRetryTimeoutMillis = maxRetryTimeoutMillis;
@ -98,6 +99,13 @@ public final class RestClient implements Closeable {
setHosts(hosts);
}
/**
* Returns a new {@link RestClientBuilder} to help with {@link RestClient} creation.
*/
public static RestClientBuilder builder(HttpHost... hosts) {
return new RestClientBuilder(hosts);
}
/**
* Replaces the hosts that the client communicates with.
* @see HttpHost
@ -116,8 +124,9 @@ public final class RestClient implements Closeable {
}
/**
* Sends a request to the elasticsearch cluster that the current client points to.
* Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters and request body.
* Sends a request to the elasticsearch cluster that the client points to and waits for the corresponding response
* to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters
* and request body.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
@ -128,12 +137,12 @@ public final class RestClient implements Closeable {
* @throws ResponseException in case elasticsearch responded with a status code that indicated an error
*/
public Response performRequest(String method, String endpoint, Header... headers) throws IOException {
return performRequest(method, endpoint, Collections.<String, String>emptyMap(), null, headers);
return performRequest(method, endpoint, Collections.<String, String>emptyMap(), (HttpEntity)null, headers);
}
/**
* Sends a request to the elasticsearch cluster that the current client points to.
* Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body.
* Sends a request to the elasticsearch cluster that the client points to and waits for the corresponding response
* to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
@ -145,15 +154,14 @@ public final class RestClient implements Closeable {
* @throws ResponseException in case elasticsearch responded with a status code that indicated an error
*/
public Response performRequest(String method, String endpoint, Map<String, String> params, Header... headers) throws IOException {
return performRequest(method, endpoint, params, null, headers);
return performRequest(method, endpoint, params, (HttpEntity)null, headers);
}
/**
* Sends a request to the elasticsearch cluster that the current client points to.
* Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain
* amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures,
* the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried
* till one responds or none of them does, in which case an {@link IOException} will be thrown.
* Sends a request to the elasticsearch cluster that the client points to and waits for the corresponding response
* to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumer, Header...)}
* which doesn't require specifying an {@link HttpAsyncResponseConsumer} instance, {@link HeapBufferedAsyncResponseConsumer}
* will be used to consume the response body.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
@ -167,72 +175,183 @@ public final class RestClient implements Closeable {
*/
public Response performRequest(String method, String endpoint, Map<String, String> params,
HttpEntity entity, Header... headers) throws IOException {
HttpAsyncResponseConsumer<HttpResponse> responseConsumer = new HeapBufferedAsyncResponseConsumer();
return performRequest(method, endpoint, params, entity, responseConsumer, headers);
}
/**
* Sends a request to the elasticsearch cluster that the client points to. Blocks until the request is completed and returns
* its response or fails by throwing an exception. Selects a host out of the provided ones in a round-robin fashion. Failing hosts
* are marked dead and retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times
* they previously failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead
* nodes that deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
* @param params the query_string parameters
* @param entity the body of the request, null if not applicable
* @param responseConsumer the {@link HttpAsyncResponseConsumer} callback. Controls how the response
* body gets streamed from a non-blocking HTTP connection on the client side.
* @param headers the optional request headers
* @return the response returned by elasticsearch
* @throws IOException in case of a problem or the connection was aborted
* @throws ClientProtocolException in case of an http protocol error
* @throws ResponseException in case elasticsearch responded with a status code that indicated an error
*/
public Response performRequest(String method, String endpoint, Map<String, String> params,
HttpEntity entity, HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
Header... headers) throws IOException {
SyncResponseListener listener = new SyncResponseListener(maxRetryTimeoutMillis);
performRequest(method, endpoint, params, entity, responseConsumer, listener, headers);
return listener.get();
}
/**
* Sends a request to the elasticsearch cluster that the client points to. Doesn't wait for the response, instead
* the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to
* {@link #performRequest(String, String, Map, HttpEntity, ResponseListener, Header...)} but without parameters and request body.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
* @param responseListener the {@link ResponseListener} to notify when the request is completed or fails
* @param headers the optional request headers
*/
public void performRequest(String method, String endpoint, ResponseListener responseListener, Header... headers) {
performRequest(method, endpoint, Collections.<String, String>emptyMap(), null, responseListener, headers);
}
/**
* Sends a request to the elasticsearch cluster that the client points to. Doesn't wait for the response, instead
* the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to
* {@link #performRequest(String, String, Map, HttpEntity, ResponseListener, Header...)} but without request body.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
* @param params the query_string parameters
* @param responseListener the {@link ResponseListener} to notify when the request is completed or fails
* @param headers the optional request headers
*/
public void performRequest(String method, String endpoint, Map<String, String> params,
ResponseListener responseListener, Header... headers) {
performRequest(method, endpoint, params, null, responseListener, headers);
}
/**
* Sends a request to the elasticsearch cluster that the client points to. Doesn't wait for the response, instead
* the provided {@link ResponseListener} will be notified upon completion or failure.
* Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumer, ResponseListener, Header...)}
* which doesn't require specifying an {@link HttpAsyncResponseConsumer} instance, {@link HeapBufferedAsyncResponseConsumer}
* will be used to consume the response body.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
* @param params the query_string parameters
* @param entity the body of the request, null if not applicable
* @param responseListener the {@link ResponseListener} to notify when the request is completed or fails
* @param headers the optional request headers
*/
public void performRequest(String method, String endpoint, Map<String, String> params,
HttpEntity entity, ResponseListener responseListener, Header... headers) {
HttpAsyncResponseConsumer<HttpResponse> responseConsumer = new HeapBufferedAsyncResponseConsumer();
performRequest(method, endpoint, params, entity, responseConsumer, responseListener, headers);
}
/**
* Sends a request to the elasticsearch cluster that the client points to. The request is executed asynchronously
* and the provided {@link ResponseListener} gets notified upon request completion or failure.
* Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain
* amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures,
* the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried
* until one responds or none of them does, in which case an {@link IOException} will be thrown.
*
* @param method the http method
* @param endpoint the path of the request (without host and port)
* @param params the query_string parameters
* @param entity the body of the request, null if not applicable
* @param responseConsumer the {@link HttpAsyncResponseConsumer} callback. Controls how the response
* body gets streamed from a non-blocking HTTP connection on the client side.
* @param responseListener the {@link ResponseListener} to notify when the request is completed or fails
* @param headers the optional request headers
*/
public void performRequest(String method, String endpoint, Map<String, String> params,
HttpEntity entity, HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
ResponseListener responseListener, Header... headers) {
URI uri = buildUri(endpoint, params);
HttpRequestBase request = createHttpRequest(method, uri, entity);
setHeaders(request, headers);
//we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
long retryTimeoutMillis = Math.round(this.maxRetryTimeoutMillis / (float)100 * 98);
IOException lastSeenException = null;
FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener);
long startTime = System.nanoTime();
for (HttpHost host : nextHost()) {
if (lastSeenException != null) {
//in case we are retrying, check whether maxRetryTimeout has been reached, in which case an exception will be thrown
long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
long timeout = retryTimeoutMillis - timeElapsedMillis;
if (timeout <= 0) {
IOException retryTimeoutException = new IOException(
"request retries exceeded max retry timeout [" + retryTimeoutMillis + "]");
retryTimeoutException.addSuppressed(lastSeenException);
throw retryTimeoutException;
performRequest(startTime, nextHost().iterator(), request, responseConsumer, failureTrackingResponseListener);
}
private void performRequest(final long startTime, final Iterator<HttpHost> hosts, final HttpRequestBase request,
final HttpAsyncResponseConsumer<HttpResponse> responseConsumer,
final FailureTrackingResponseListener listener) {
final HttpHost host = hosts.next();
//we stream the request body if the entity allows for it
HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request);
client.execute(requestProducer, responseConsumer, new FutureCallback<HttpResponse>() {
@Override
public void completed(HttpResponse httpResponse) {
try {
RequestLogger.logResponse(logger, request, host, httpResponse);
int statusCode = httpResponse.getStatusLine().getStatusCode();
Response response = new Response(request.getRequestLine(), host, httpResponse);
if (isSuccessfulResponse(request.getMethod(), statusCode)) {
onResponse(host);
listener.onSuccess(response);
} else {
ResponseException responseException = new ResponseException(response);
if (isRetryStatus(statusCode)) {
//mark host dead and retry against next one
onFailure(host);
retryIfPossible(responseException, hosts, request);
} else {
//mark host alive and don't retry, as the error should be a request problem
onResponse(host);
listener.onDefinitiveFailure(responseException);
}
}
} catch(Exception e) {
listener.onDefinitiveFailure(e);
}
//also reset the request to make it reusable for the next attempt
request.reset();
}
CloseableHttpResponse httpResponse;
try {
httpResponse = client.execute(host, request);
} catch(IOException e) {
RequestLogger.logFailedRequest(logger, request, host, e);
onFailure(host);
lastSeenException = addSuppressedException(lastSeenException, e);
continue;
}
Response response = new Response(request.getRequestLine(), host, httpResponse);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
RequestLogger.logResponse(logger, request, host, httpResponse);
onResponse(host);
return response;
}
RequestLogger.logResponse(logger, request, host, httpResponse);
String responseBody;
try {
if (response.getEntity() == null) {
responseBody = null;
} else {
responseBody = EntityUtils.toString(response.getEntity());
}
} finally {
response.close();
}
lastSeenException = addSuppressedException(lastSeenException, new ResponseException(response, responseBody));
switch(statusCode) {
case 502:
case 503:
case 504:
//mark host dead and retry against next one
@Override
public void failed(Exception failure) {
try {
RequestLogger.logFailedRequest(logger, request, host, failure);
onFailure(host);
break;
default:
//mark host alive and don't retry, as the error should be a request problem
onResponse(host);
throw lastSeenException;
retryIfPossible(failure, hosts, request);
} catch(Exception e) {
listener.onDefinitiveFailure(e);
}
}
}
//we get here only when we tried all nodes and they all failed
assert lastSeenException != null;
throw lastSeenException;
private void retryIfPossible(Exception exception, Iterator<HttpHost> hosts, HttpRequestBase request) {
if (hosts.hasNext()) {
//in case we are retrying, check whether maxRetryTimeout has been reached
long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
long timeout = maxRetryTimeoutMillis - timeElapsedMillis;
if (timeout <= 0) {
IOException retryTimeoutException = new IOException(
"request retries exceeded max retry timeout [" + maxRetryTimeoutMillis + "]");
listener.onDefinitiveFailure(retryTimeoutException);
} else {
listener.trackFailure(exception);
request.reset();
performRequest(startTime, hosts, request, responseConsumer, listener);
}
} else {
listener.onDefinitiveFailure(exception);
}
}
@Override
public void cancelled() {
listener.onDefinitiveFailure(new ExecutionException("request was cancelled", null));
}
});
}
private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) {
@ -247,38 +366,43 @@ public final class RestClient implements Closeable {
}
/**
* Returns an iterator of hosts to be used for a request call.
* Ideally, the first host is retrieved from the iterator and used successfully for the request.
* Otherwise, after each failure the next host should be retrieved from the iterator so that the request can be retried till
* the iterator is exhausted. The maximum total of attempts is equal to the number of hosts that are available in the iterator.
* The iterator returned will never be empty, rather an {@link IllegalStateException} in case there are no hosts.
* In case there are no healthy hosts available, or dead ones to be be retried, one dead host gets returned.
* Returns an {@link Iterable} of hosts to be used for a request call.
* Ideally, the first host is retrieved from the iterable and used successfully for the request.
* Otherwise, after each failure the next host has to be retrieved from the iterator so that the request can be retried until
* there are no more hosts available to retry against. The maximum total of attempts is equal to the number of hosts in the iterable.
* The iterator returned will never be empty. In case there are no healthy hosts available, or dead ones to be be retried,
* one dead host gets returned so that it can be retried.
*/
private Iterable<HttpHost> nextHost() {
Set<HttpHost> filteredHosts = new HashSet<>(hosts);
for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
filteredHosts.remove(entry.getKey());
}
}
if (filteredHosts.isEmpty()) {
//last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried
List<Map.Entry<HttpHost, DeadHostState>> sortedHosts = new ArrayList<>(blacklist.entrySet());
Collections.sort(sortedHosts, new Comparator<Map.Entry<HttpHost, DeadHostState>>() {
@Override
public int compare(Map.Entry<HttpHost, DeadHostState> o1, Map.Entry<HttpHost, DeadHostState> o2) {
return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos());
Collection<HttpHost> nextHosts = Collections.emptySet();
do {
Set<HttpHost> filteredHosts = new HashSet<>(hosts);
for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
filteredHosts.remove(entry.getKey());
}
});
HttpHost deadHost = sortedHosts.get(0).getKey();
logger.trace("resurrecting host [" + deadHost + "]");
return Collections.singleton(deadHost);
}
List<HttpHost> rotatedHosts = new ArrayList<>(filteredHosts);
Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement());
return rotatedHosts;
}
if (filteredHosts.isEmpty()) {
//last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried
List<Map.Entry<HttpHost, DeadHostState>> sortedHosts = new ArrayList<>(blacklist.entrySet());
if (sortedHosts.size() > 0) {
Collections.sort(sortedHosts, new Comparator<Map.Entry<HttpHost, DeadHostState>>() {
@Override
public int compare(Map.Entry<HttpHost, DeadHostState> o1, Map.Entry<HttpHost, DeadHostState> o2) {
return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos());
}
});
HttpHost deadHost = sortedHosts.get(0).getKey();
logger.trace("resurrecting host [" + deadHost + "]");
nextHosts = Collections.singleton(deadHost);
}
} else {
List<HttpHost> rotatedHosts = new ArrayList<>(filteredHosts);
Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement());
nextHosts = rotatedHosts;
}
} while(nextHosts.isEmpty());
return nextHosts;
}
/**
@ -316,7 +440,21 @@ public final class RestClient implements Closeable {
client.close();
}
private static IOException addSuppressedException(IOException suppressedException, IOException currentException) {
private static boolean isSuccessfulResponse(String method, int statusCode) {
return statusCode < 300 || (HttpHead.METHOD_NAME.equals(method) && statusCode == 404);
}
private static boolean isRetryStatus(int statusCode) {
switch(statusCode) {
case 502:
case 503:
case 504:
return true;
}
return false;
}
private static Exception addSuppressedException(Exception suppressedException, Exception currentException) {
if (suppressedException != null) {
currentException.addSuppressed(suppressedException);
}
@ -373,156 +511,114 @@ public final class RestClient implements Closeable {
}
/**
* Returns a new {@link Builder} to help with {@link RestClient} creation.
* Listener used in any async call to wrap the provided user listener (or SyncResponseListener in sync calls).
* Allows to track potential failures coming from the different retry attempts and returning to the original listener
* only when we got a response (successful or not to be retried) or there are no hosts to retry against.
*/
public static Builder builder(HttpHost... hosts) {
return new Builder(hosts);
}
static class FailureTrackingResponseListener {
private final ResponseListener responseListener;
private volatile Exception exception;
/**
* Rest client builder. Helps creating a new {@link RestClient}.
*/
public static final class Builder {
public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000;
public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 10000;
public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS;
public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
private static final Header[] EMPTY_HEADERS = new Header[0];
private final HttpHost[] hosts;
private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS;
private Header[] defaultHeaders = EMPTY_HEADERS;
private FailureListener failureListener;
private HttpClientConfigCallback httpClientConfigCallback;
private RequestConfigCallback requestConfigCallback;
/**
* Creates a new builder instance and sets the hosts that the client will send requests to.
*/
private Builder(HttpHost... hosts) {
if (hosts == null || hosts.length == 0) {
throw new IllegalArgumentException("no hosts provided");
}
this.hosts = hosts;
FailureTrackingResponseListener(ResponseListener responseListener) {
this.responseListener = responseListener;
}
/**
* Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request.
* {@link #DEFAULT_MAX_RETRY_TIMEOUT_MILLIS} if not specified.
*
* @throws IllegalArgumentException if maxRetryTimeoutMillis is not greater than 0
* Notifies the caller of a response through the wrapped listener
*/
public Builder setMaxRetryTimeoutMillis(int maxRetryTimeoutMillis) {
if (maxRetryTimeoutMillis <= 0) {
throw new IllegalArgumentException("maxRetryTimeoutMillis must be greater than 0");
}
this.maxRetryTimeout = maxRetryTimeoutMillis;
return this;
void onSuccess(Response response) {
responseListener.onSuccess(response);
}
/**
* Sets the default request headers, to be used sent with every request unless overridden on a per request basis
* Tracks one last definitive failure and returns to the caller by notifying the wrapped listener
*/
public Builder setDefaultHeaders(Header[] defaultHeaders) {
Objects.requireNonNull(defaultHeaders, "defaultHeaders must not be null");
for (Header defaultHeader : defaultHeaders) {
Objects.requireNonNull(defaultHeader, "default header must not be null");
}
this.defaultHeaders = defaultHeaders;
return this;
void onDefinitiveFailure(Exception exception) {
trackFailure(exception);
responseListener.onFailure(this.exception);
}
/**
* Sets the {@link FailureListener} to be notified for each request failure
* Tracks an exception, which caused a retry hence we should not return yet to the caller
*/
public Builder setFailureListener(FailureListener failureListener) {
Objects.requireNonNull(failureListener, "failureListener must not be null");
this.failureListener = failureListener;
return this;
}
/**
* Sets the {@link HttpClientConfigCallback} to be used to customize http client configuration
*/
public Builder setHttpClientConfigCallback(HttpClientConfigCallback httpClientConfigCallback) {
Objects.requireNonNull(httpClientConfigCallback, "httpClientConfigCallback must not be null");
this.httpClientConfigCallback = httpClientConfigCallback;
return this;
}
/**
* Sets the {@link RequestConfigCallback} to be used to customize http client configuration
*/
public Builder setRequestConfigCallback(RequestConfigCallback requestConfigCallback) {
Objects.requireNonNull(requestConfigCallback, "requestConfigCallback must not be null");
this.requestConfigCallback = requestConfigCallback;
return this;
}
/**
* Creates a new {@link RestClient} based on the provided configuration.
*/
public RestClient build() {
if (failureListener == null) {
failureListener = new FailureListener();
}
CloseableHttpClient httpClient = createHttpClient();
return new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, failureListener);
}
private CloseableHttpClient createHttpClient() {
//default timeouts are all infinite
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS)
.setSocketTimeout(DEFAULT_SOCKET_TIMEOUT_MILLIS)
.setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS);
if (requestConfigCallback != null) {
requestConfigCallback.customizeRequestConfig(requestConfigBuilder);
}
RequestConfig requestConfig = requestConfigBuilder.build();
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
//default settings may be too constraining
connectionManager.setDefaultMaxPerRoute(10);
connectionManager.setMaxTotal(30);
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig);
if (httpClientConfigCallback != null) {
httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
}
return httpClientBuilder.build();
void trackFailure(Exception exception) {
this.exception = addSuppressedException(this.exception, exception);
}
}
/**
* Callback used the default {@link RequestConfig} being set to the {@link CloseableHttpClient}
* @see HttpClientBuilder#setDefaultRequestConfig
* Listener used in any sync performRequest calls, it waits for a response or an exception back up to a timeout
*/
public interface RequestConfigCallback {
/**
* Allows to customize the {@link RequestConfig} that will be used with each request.
* It is common to customize the different timeout values through this method without losing any other useful default
* value that the {@link RestClient.Builder} internally sets.
*/
void customizeRequestConfig(RequestConfig.Builder requestConfigBuilder);
}
static class SyncResponseListener implements ResponseListener {
private final CountDownLatch latch = new CountDownLatch(1);
private final AtomicReference<Response> response = new AtomicReference<>();
private final AtomicReference<Exception> exception = new AtomicReference<>();
private final long timeout;
SyncResponseListener(long timeout) {
assert timeout > 0;
this.timeout = timeout;
}
@Override
public void onSuccess(Response response) {
Objects.requireNonNull(response, "response must not be null");
boolean wasResponseNull = this.response.compareAndSet(null, response);
if (wasResponseNull == false) {
throw new IllegalStateException("response is already set");
}
latch.countDown();
}
@Override
public void onFailure(Exception exception) {
Objects.requireNonNull(exception, "exception must not be null");
boolean wasExceptionNull = this.exception.compareAndSet(null, exception);
if (wasExceptionNull == false) {
throw new IllegalStateException("exception is already set");
}
latch.countDown();
}
/**
* Callback used to customize the {@link CloseableHttpClient} instance used by a {@link RestClient} instance.
* Allows to customize default {@link RequestConfig} being set to the client and any parameter that
* can be set through {@link HttpClientBuilder}
*/
public interface HttpClientConfigCallback {
/**
* Allows to customize the {@link CloseableHttpClient} being created and used by the {@link RestClient}.
* It is common to customzie the default {@link org.apache.http.client.CredentialsProvider} through this method,
* without losing any other useful default value that the {@link RestClient.Builder} internally sets.
* Also useful to setup ssl through {@link SSLSocketFactoryHttpConfigCallback}.
* Waits (up to a timeout) for some result of the request: either a response, or an exception.
*/
void customizeHttpClient(HttpClientBuilder httpClientBuilder);
Response get() throws IOException {
try {
//providing timeout is just a safety measure to prevent everlasting waits
//the different client timeouts should already do their jobs
if (latch.await(timeout, TimeUnit.MILLISECONDS) == false) {
throw new IOException("listener timeout after waiting for [" + timeout + "] ms");
}
} catch (InterruptedException e) {
throw new RuntimeException("thread waiting for the response was interrupted", e);
}
Exception exception = this.exception.get();
Response response = this.response.get();
if (exception != null) {
if (response != null) {
IllegalStateException e = new IllegalStateException("response and exception are unexpectedly set at the same time");
e.addSuppressed(exception);
throw e;
}
//try and leave the exception untouched as much as possible but we don't want to just add throws Exception clause everywhere
if (exception instanceof IOException) {
throw (IOException) exception;
}
if (exception instanceof RuntimeException){
throw (RuntimeException) exception;
}
throw new RuntimeException("error while performing request", exception);
}
if (response == null) {
throw new IllegalStateException("response not set and no exception caught either");
}
return response;
}
}
/**
@ -533,7 +629,7 @@ public final class RestClient implements Closeable {
/**
* Notifies that the host provided as argument has just failed
*/
public void onFailure(HttpHost host) throws IOException {
public void onFailure(HttpHost host) {
}
}

179
client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java Normal file
View File

@ -0,0 +1,179 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.nio.conn.SchemeIOSessionStrategy;
import java.util.Objects;
/**
* Helps creating a new {@link RestClient}. Allows to set the most common http client configuration options when internally
* creating the underlying {@link org.apache.http.nio.client.HttpAsyncClient}. Also allows to provide an externally created
* {@link org.apache.http.nio.client.HttpAsyncClient} in case additional customization is needed.
*/
public final class RestClientBuilder {
public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000;
public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 10000;
public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS;
public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
public static final int DEFAULT_MAX_CONN_PER_ROUTE = 10;
public static final int DEFAULT_MAX_CONN_TOTAL = 30;
private static final Header[] EMPTY_HEADERS = new Header[0];
private final HttpHost[] hosts;
private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS;
private Header[] defaultHeaders = EMPTY_HEADERS;
private RestClient.FailureListener failureListener;
private HttpClientConfigCallback httpClientConfigCallback;
private RequestConfigCallback requestConfigCallback;
/**
* Creates a new builder instance and sets the hosts that the client will send requests to.
*/
RestClientBuilder(HttpHost... hosts) {
if (hosts == null || hosts.length == 0) {
throw new IllegalArgumentException("no hosts provided");
}
for (HttpHost host : hosts) {
Objects.requireNonNull(host, "host cannot be null");
}
this.hosts = hosts;
}
/**
* Sets the default request headers, which will be sent along with each request
*/
public RestClientBuilder setDefaultHeaders(Header[] defaultHeaders) {
Objects.requireNonNull(defaultHeaders, "defaultHeaders must not be null");
for (Header defaultHeader : defaultHeaders) {
Objects.requireNonNull(defaultHeader, "default header must not be null");
}
this.defaultHeaders = defaultHeaders;
return this;
}
/**
* Sets the {@link RestClient.FailureListener} to be notified for each request failure
*/
public RestClientBuilder setFailureListener(RestClient.FailureListener failureListener) {
Objects.requireNonNull(failureListener, "failureListener must not be null");
this.failureListener = failureListener;
return this;
}
/**
* Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request.
* {@link #DEFAULT_MAX_RETRY_TIMEOUT_MILLIS} if not specified.
*
* @throws IllegalArgumentException if maxRetryTimeoutMillis is not greater than 0
*/
public RestClientBuilder setMaxRetryTimeoutMillis(int maxRetryTimeoutMillis) {
if (maxRetryTimeoutMillis <= 0) {
throw new IllegalArgumentException("maxRetryTimeoutMillis must be greater than 0");
}
this.maxRetryTimeout = maxRetryTimeoutMillis;
return this;
}
/**
* Sets the {@link HttpClientConfigCallback} to be used to customize http client configuration
*/
public RestClientBuilder setHttpClientConfigCallback(HttpClientConfigCallback httpClientConfigCallback) {
Objects.requireNonNull(httpClientConfigCallback, "httpClientConfigCallback must not be null");
this.httpClientConfigCallback = httpClientConfigCallback;
return this;
}
/**
* Sets the {@link RequestConfigCallback} to be used to customize http client configuration
*/
public RestClientBuilder setRequestConfigCallback(RequestConfigCallback requestConfigCallback) {
Objects.requireNonNull(requestConfigCallback, "requestConfigCallback must not be null");
this.requestConfigCallback = requestConfigCallback;
return this;
}
/**
* Creates a new {@link RestClient} based on the provided configuration.
*/
public RestClient build() {
if (failureListener == null) {
failureListener = new RestClient.FailureListener();
}
CloseableHttpAsyncClient httpClient = createHttpClient();
RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, failureListener);
httpClient.start();
return restClient;
}
private CloseableHttpAsyncClient createHttpClient() {
//default timeouts are all infinite
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS)
.setSocketTimeout(DEFAULT_SOCKET_TIMEOUT_MILLIS)
.setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS);
if (requestConfigCallback != null) {
requestConfigBuilder = requestConfigCallback.customizeRequestConfig(requestConfigBuilder);
}
HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClientBuilder.create().setDefaultRequestConfig(requestConfigBuilder.build())
//default settings for connection pooling may be too constraining
.setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL);
if (httpClientConfigCallback != null) {
httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
}
return httpClientBuilder.build();
}
/**
* Callback used the default {@link RequestConfig} being set to the {@link CloseableHttpClient}
* @see HttpClientBuilder#setDefaultRequestConfig
*/
public interface RequestConfigCallback {
/**
* Allows to customize the {@link RequestConfig} that will be used with each request.
* It is common to customize the different timeout values through this method without losing any other useful default
* value that the {@link RestClientBuilder} internally sets.
*/
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder);
}
/**
* Callback used to customize the {@link CloseableHttpClient} instance used by a {@link RestClient} instance.
* Allows to customize default {@link RequestConfig} being set to the client and any parameter that
* can be set through {@link HttpClientBuilder}
*/
public interface HttpClientConfigCallback {
/**
* Allows to customize the {@link CloseableHttpAsyncClient} being created and used by the {@link RestClient}.
* Commonly used to customize the default {@link org.apache.http.client.CredentialsProvider} for authentication
* or the {@link SchemeIOSessionStrategy} for communication through ssl without losing any other useful default
* value that the {@link RestClientBuilder} internally sets, like connection pooling.
*/
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder);
}
}

53
client/rest/src/main/java/org/elasticsearch/client/SSLSocketFactoryHttpConfigCallback.java
View File

@ -1,53 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
/**
* Helps configuring the http client when needing to communicate over ssl. It effectively replaces the connection manager
* with one that has ssl properly configured thanks to the provided {@link SSLConnectionSocketFactory}.
*/
public class SSLSocketFactoryHttpConfigCallback implements RestClient.HttpClientConfigCallback {
private final SSLConnectionSocketFactory sslSocketFactory;
public SSLSocketFactoryHttpConfigCallback(SSLConnectionSocketFactory sslSocketFactory) {
this.sslSocketFactory = sslSocketFactory;
}
@Override
public void customizeHttpClient(HttpClientBuilder httpClientBuilder) {
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", sslSocketFactory).build();
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
//default settings may be too constraining
connectionManager.setDefaultMaxPerRoute(10);
connectionManager.setMaxTotal(30);
httpClientBuilder.setConnectionManager(connectionManager);
}
}

107
client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.RequestLine;
import org.apache.http.StatusLine;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicRequestLine;
import org.apache.http.message.BasicStatusLine;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
public class FailureTrackingResponseListenerTests extends RestClientTestCase {
public void testOnSuccess() {
MockResponseListener responseListener = new MockResponseListener();
RestClient.FailureTrackingResponseListener listener = new RestClient.FailureTrackingResponseListener(responseListener);
}
public void testOnFailure() {
MockResponseListener responseListener = new MockResponseListener();
RestClient.FailureTrackingResponseListener listener = new RestClient.FailureTrackingResponseListener(responseListener);
int numIters = randomIntBetween(1, 10);
Exception[] expectedExceptions = new Exception[numIters];
for (int i = 0; i < numIters; i++) {
RuntimeException runtimeException = new RuntimeException("test" + i);
expectedExceptions[i] = runtimeException;
listener.trackFailure(runtimeException);
assertNull(responseListener.response.get());
assertNull(responseListener.exception.get());
}
if (randomBoolean()) {
Response response = mockResponse();
listener.onSuccess(response);
assertSame(response, responseListener.response.get());
assertNull(responseListener.exception.get());
} else {
RuntimeException runtimeException = new RuntimeException("definitive");
listener.onDefinitiveFailure(runtimeException);
assertNull(responseListener.response.get());
Throwable exception = responseListener.exception.get();
assertSame(runtimeException, exception);
int i = numIters - 1;
do {
assertNotNull(exception.getSuppressed());
assertEquals(1, exception.getSuppressed().length);
assertSame(expectedExceptions[i--], exception.getSuppressed()[0]);
exception = exception.getSuppressed()[0];
} while(i >= 0);
}
}
private static class MockResponseListener implements ResponseListener {
private final AtomicReference<Response> response = new AtomicReference<>();
private final AtomicReference<Exception> exception = new AtomicReference<>();
@Override
public void onSuccess(Response response) {
if (this.response.compareAndSet(null, response) == false) {
throw new IllegalStateException("onSuccess was called multiple times");
}
}
@Override
public void onFailure(Exception exception) {
if (this.exception.compareAndSet(null, exception) == false) {
throw new IllegalStateException("onFailure was called multiple times");
}
}
}
private static Response mockResponse() {
ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
RequestLine requestLine = new BasicRequestLine("GET", "/", protocolVersion);
StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK");
HttpResponse httpResponse = new BasicHttpResponse(statusLine);
return new Response(requestLine, new HttpHost("localhost", 9200), httpResponse);
}
}

114
client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.ContentTooLongException;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.entity.BasicHttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicStatusLine;
import org.apache.http.nio.ContentDecoder;
import org.apache.http.nio.IOControl;
import org.apache.http.protocol.HttpContext;
import static org.elasticsearch.client.HeapBufferedAsyncResponseConsumer.DEFAULT_BUFFER_LIMIT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
public class HeapBufferedAsyncResponseConsumerTests extends RestClientTestCase {
//maximum buffer that this test ends up allocating is 50MB
private static final int MAX_TEST_BUFFER_SIZE = 50 * 1024 * 1024;
public void testResponseProcessing() throws Exception {
ContentDecoder contentDecoder = mock(ContentDecoder.class);
IOControl ioControl = mock(IOControl.class);
HttpContext httpContext = mock(HttpContext.class);
HeapBufferedAsyncResponseConsumer consumer = spy(new HeapBufferedAsyncResponseConsumer());
ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK");
HttpResponse httpResponse = new BasicHttpResponse(statusLine);
httpResponse.setEntity(new StringEntity("test"));
//everything goes well
consumer.responseReceived(httpResponse);
consumer.consumeContent(contentDecoder, ioControl);
consumer.responseCompleted(httpContext);
verify(consumer).releaseResources();
verify(consumer).buildResult(httpContext);
assertTrue(consumer.isDone());
assertSame(httpResponse, consumer.getResult());
consumer.responseCompleted(httpContext);
verify(consumer, times(1)).releaseResources();
verify(consumer, times(1)).buildResult(httpContext);
}
public void testDefaultBufferLimit() throws Exception {
HeapBufferedAsyncResponseConsumer consumer = new HeapBufferedAsyncResponseConsumer();
bufferLimitTest(consumer, DEFAULT_BUFFER_LIMIT);
}
public void testConfiguredBufferLimit() throws Exception {
try {
new HeapBufferedAsyncResponseConsumer(randomIntBetween(Integer.MIN_VALUE, 0));
} catch(IllegalArgumentException e) {
assertEquals("bufferLimit must be greater than 0", e.getMessage());
}
try {
new HeapBufferedAsyncResponseConsumer(0);
} catch(IllegalArgumentException e) {
assertEquals("bufferLimit must be greater than 0", e.getMessage());
}
int bufferLimit = randomIntBetween(1, MAX_TEST_BUFFER_SIZE - 100);
HeapBufferedAsyncResponseConsumer consumer = new HeapBufferedAsyncResponseConsumer(bufferLimit);
bufferLimitTest(consumer, bufferLimit);
}
private static void bufferLimitTest(HeapBufferedAsyncResponseConsumer consumer, int bufferLimit) throws Exception {
ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK");
consumer.onResponseReceived(new BasicHttpResponse(statusLine));
BasicHttpEntity entity = new BasicHttpEntity();
entity.setContentLength(randomInt(bufferLimit));
consumer.onEntityEnclosed(entity, ContentType.APPLICATION_JSON);
entity.setContentLength(randomIntBetween(bufferLimit + 1, MAX_TEST_BUFFER_SIZE));
try {
consumer.onEntityEnclosed(entity, ContentType.APPLICATION_JSON);
} catch(ContentTooLongException e) {
assertEquals("entity content is too long [" + entity.getContentLength() +
"] for the configured buffer limit [" + bufferLimit + "]", e.getMessage());
}
}
}

9
client/rest/src/test/java/org/elasticsearch/client/TrackingFailureListener.java → client/rest/src/test/java/org/elasticsearch/client/HostsTrackingFailureListener.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.client;
import org.apache.http.HttpHost;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
@ -30,13 +29,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
/**
* {@link org.elasticsearch.client.RestClient.FailureListener} impl that allows to track when it gets called
* {@link org.elasticsearch.client.RestClient.FailureListener} impl that allows to track when it gets called for which host.
*/
class TrackingFailureListener extends RestClient.FailureListener {
private Set<HttpHost> hosts = new HashSet<>();
class HostsTrackingFailureListener extends RestClient.FailureListener {
private volatile Set<HttpHost> hosts = new HashSet<>();
@Override
public void onFailure(HttpHost host) throws IOException {
public void onFailure(HttpHost host) {
hosts.add(host);
}

22
client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
View File

@ -35,6 +35,8 @@ import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicStatusLine;
import org.apache.http.nio.entity.NByteArrayEntity;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.http.util.EntityUtils;
import java.io.ByteArrayInputStream;
@ -97,14 +99,24 @@ public class RequestLoggerTests extends RestClientTestCase {
expected += " -d '" + requestBody + "'";
HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
HttpEntity entity;
if (getRandom().nextBoolean()) {
entity = new StringEntity(requestBody, StandardCharsets.UTF_8);
} else {
entity = new InputStreamEntity(new ByteArrayInputStream(requestBody.getBytes(StandardCharsets.UTF_8)));
switch(RandomInts.randomIntBetween(getRandom(), 0, 3)) {
case 0:
entity = new StringEntity(requestBody, StandardCharsets.UTF_8);
break;
case 1:
entity = new InputStreamEntity(new ByteArrayInputStream(requestBody.getBytes(StandardCharsets.UTF_8)));
break;
case 2:
entity = new NStringEntity(requestBody, StandardCharsets.UTF_8);
break;
case 3:
entity = new NByteArrayEntity(requestBody.getBytes(StandardCharsets.UTF_8));
break;
default:
throw new UnsupportedOperationException();
}
enclosingRequest.setEntity(entity);
}
String traceRequest = RequestLogger.buildTraceRequest(request, host);
assertThat(traceRequest, equalTo(expected));
if (hasBody) {

20
client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
View File

@ -23,7 +23,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
import java.io.IOException;
@ -50,12 +50,16 @@ public class RestClientBuilderTests extends RestClientTestCase {
}
try {
RestClient.builder(new HttpHost[]{new HttpHost("localhost", 9200), null}).build();
RestClient.builder(new HttpHost("localhost", 9200), null);
fail("should have failed");
} catch(NullPointerException e) {
assertEquals("host cannot be null", e.getMessage());
}
try (RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build()) {
assertNotNull(restClient);
}
try {
RestClient.builder(new HttpHost("localhost", 9200))
.setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
@ -104,18 +108,20 @@ public class RestClientBuilderTests extends RestClientTestCase {
for (int i = 0; i < numNodes; i++) {
hosts[i] = new HttpHost("localhost", 9200 + i);
}
RestClient.Builder builder = RestClient.builder(hosts);
RestClientBuilder builder = RestClient.builder(hosts);
if (getRandom().nextBoolean()) {
builder.setHttpClientConfigCallback(new RestClient.HttpClientConfigCallback() {
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public void customizeHttpClient(HttpClientBuilder httpClientBuilder) {
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
return httpClientBuilder;
}
});
}
if (getRandom().nextBoolean()) {
builder.setRequestConfigCallback(new RestClient.RequestConfigCallback() {
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
public void customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
return requestConfigBuilder;
}
});
}

85
client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
View File

@ -27,6 +27,7 @@ import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicHeader;
@ -47,6 +48,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
@ -140,10 +144,10 @@ public class RestClientIntegTests extends RestClientTestCase {
* to set/add headers to the {@link org.apache.http.client.HttpClient}.
* Exercises the test http server ability to send back whatever headers it received.
*/
public void testHeaders() throws Exception {
public void testHeaders() throws IOException {
for (String method : getHttpMethods()) {
Set<String> standardHeaders = new HashSet<>(
Arrays.asList("Accept-encoding", "Connection", "Host", "User-agent", "Date"));
Arrays.asList("Connection", "Host", "User-agent", "Date"));
if (method.equals("HEAD") == false) {
standardHeaders.add("Content-length");
}
@ -162,9 +166,9 @@ public class RestClientIntegTests extends RestClientTestCase {
int statusCode = randomStatusCode(getRandom());
Response esResponse;
try (Response response = restClient.performRequest(method, "/" + statusCode,
Collections.<String, String>emptyMap(), null, headers)) {
esResponse = response;
try {
esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(),
(HttpEntity)null, headers);
} catch(ResponseException e) {
esResponse = e.getResponse();
}
@ -188,7 +192,7 @@ public class RestClientIntegTests extends RestClientTestCase {
* out of the box by {@link org.apache.http.client.HttpClient}.
* Exercises the test http server ability to send back whatever body it received.
*/
public void testDeleteWithBody() throws Exception {
public void testDeleteWithBody() throws IOException {
bodyTest("DELETE");
}
@ -197,25 +201,74 @@ public class RestClientIntegTests extends RestClientTestCase {
* out of the box by {@link org.apache.http.client.HttpClient}.
* Exercises the test http server ability to send back whatever body it received.
*/
public void testGetWithBody() throws Exception {
public void testGetWithBody() throws IOException {
bodyTest("GET");
}
private void bodyTest(String method) throws Exception {
private void bodyTest(String method) throws IOException {
String requestBody = "{ \"field\": \"value\" }";
StringEntity entity = new StringEntity(requestBody);
Response esResponse;
String responseBody;
int statusCode = randomStatusCode(getRandom());
try (Response response = restClient.performRequest(method, "/" + statusCode,
Collections.<String, String>emptyMap(), entity)) {
responseBody = EntityUtils.toString(response.getEntity());
esResponse = response;
Response esResponse;
try {
esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), entity);
} catch(ResponseException e) {
responseBody = e.getResponseBody();
esResponse = e.getResponse();
}
assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
assertEquals(requestBody, responseBody);
assertEquals(requestBody, EntityUtils.toString(esResponse.getEntity()));
}
public void testAsyncRequests() throws Exception {
int numRequests = randomIntBetween(5, 20);
final CountDownLatch latch = new CountDownLatch(numRequests);
final List<TestResponse> responses = new CopyOnWriteArrayList<>();
for (int i = 0; i < numRequests; i++) {
final String method = RestClientTestUtil.randomHttpMethod(getRandom());
final int statusCode = randomStatusCode(getRandom());
restClient.performRequest(method, "/" + statusCode, new ResponseListener() {
@Override
public void onSuccess(Response response) {
responses.add(new TestResponse(method, statusCode, response));
latch.countDown();
}
@Override
public void onFailure(Exception exception) {
responses.add(new TestResponse(method, statusCode, exception));
latch.countDown();
}
});
}
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertEquals(numRequests, responses.size());
for (TestResponse response : responses) {
assertEquals(response.method, response.getResponse().getRequestLine().getMethod());
assertEquals(response.statusCode, response.getResponse().getStatusLine().getStatusCode());
}
}
private static class TestResponse {
private final String method;
private final int statusCode;
private final Object response;
TestResponse(String method, int statusCode, Object response) {
this.method = method;
this.statusCode = statusCode;
this.response = response;
}
Response getResponse() {
if (response instanceof Response) {
return (Response) response;
}
if (response instanceof ResponseException) {
return ((ResponseException) response).getResponse();
}
throw new AssertionError("unexpected response " + response.getClass());
}
}
}

71
client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
View File

@ -22,14 +22,17 @@ package org.elasticsearch.client;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicStatusLine;
import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
import org.junit.Before;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@ -39,6 +42,7 @@ import java.net.SocketTimeoutException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Future;
import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode;
import static org.elasticsearch.client.RestClientTestUtil.randomErrorRetryStatusCode;
@ -62,57 +66,61 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
private RestClient restClient;
private HttpHost[] httpHosts;
private TrackingFailureListener failureListener;
private HostsTrackingFailureListener failureListener;
@Before
@SuppressWarnings("unchecked")
public void createRestClient() throws IOException {
CloseableHttpClient httpClient = mock(CloseableHttpClient.class);
when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenAnswer(new Answer<CloseableHttpResponse>() {
CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
@Override
public CloseableHttpResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
HttpHost httpHost = (HttpHost) invocationOnMock.getArguments()[0];
HttpUriRequest request = (HttpUriRequest) invocationOnMock.getArguments()[1];
public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
HttpHost httpHost = requestProducer.getTarget();
FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
//return the desired status code or exception depending on the path
if (request.getURI().getPath().equals("/soe")) {
throw new SocketTimeoutException(httpHost.toString());
futureCallback.failed(new SocketTimeoutException(httpHost.toString()));
} else if (request.getURI().getPath().equals("/coe")) {
throw new ConnectTimeoutException(httpHost.toString());
futureCallback.failed(new ConnectTimeoutException(httpHost.toString()));
} else if (request.getURI().getPath().equals("/ioe")) {
throw new IOException(httpHost.toString());
futureCallback.failed(new IOException(httpHost.toString()));
} else {
int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
futureCallback.completed(new BasicHttpResponse(statusLine));
}
int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
return new CloseableBasicHttpResponse(statusLine);
return null;
}
});
int numHosts = RandomInts.randomIntBetween(getRandom(), 2, 5);
httpHosts = new HttpHost[numHosts];
for (int i = 0; i < numHosts; i++) {
httpHosts[i] = new HttpHost("localhost", 9200 + i);
}
failureListener = new TrackingFailureListener();
failureListener = new HostsTrackingFailureListener();
restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, failureListener);
}
public void testRoundRobinOkStatusCodes() throws Exception {
public void testRoundRobinOkStatusCodes() throws IOException {
int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5);
for (int i = 0; i < numIters; i++) {
Set<HttpHost> hostsSet = new HashSet<>();
Collections.addAll(hostsSet, httpHosts);
for (int j = 0; j < httpHosts.length; j++) {
int statusCode = randomOkStatusCode(getRandom());
try (Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode)) {
assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
}
Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode);
assertEquals(statusCode, response.getStatusLine().getStatusCode());
assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
}
assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
}
failureListener.assertNotCalled();
}
public void testRoundRobinNoRetryErrors() throws Exception {
public void testRoundRobinNoRetryErrors() throws IOException {
int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5);
for (int i = 0; i < numIters; i++) {
Set<HttpHost> hostsSet = new HashSet<>();
@ -120,11 +128,12 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
for (int j = 0; j < httpHosts.length; j++) {
String method = randomHttpMethod(getRandom());
int statusCode = randomErrorNoRetryStatusCode(getRandom());
try (Response response = restClient.performRequest(method, "/" + statusCode)) {
try {
Response response = restClient.performRequest(method, "/" + statusCode);
if (method.equals("HEAD") && statusCode == 404) {
//no exception gets thrown although we got a 404
assertThat(response.getStatusLine().getStatusCode(), equalTo(404));
assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
assertEquals(404, response.getStatusLine().getStatusCode());
assertEquals(statusCode, response.getStatusLine().getStatusCode());
assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
} else {
fail("request should have failed");
@ -134,7 +143,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
throw e;
}
Response response = e.getResponse();
assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
assertEquals(statusCode, response.getStatusLine().getStatusCode());
assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
assertEquals(0, e.getSuppressed().length);
}
@ -144,7 +153,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
failureListener.assertNotCalled();
}
public void testRoundRobinRetryErrors() throws Exception {
public void testRoundRobinRetryErrors() throws IOException {
String retryEndpoint = randomErrorRetryEndpoint();
try {
restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint);
@ -156,7 +165,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
failureListener.assertCalled(httpHosts);
do {
Response response = e.getResponse();
assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
assertEquals(Integer.parseInt(retryEndpoint.substring(1)), response.getStatusLine().getStatusCode());
assertTrue("host [" + response.getHost() + "] not found, most likely used multiple times",
hostsSet.remove(response.getHost()));
if (e.getSuppressed().length > 0) {
@ -223,8 +232,8 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
for (int y = 0; y < iters; y++) {
int statusCode = randomErrorNoRetryStatusCode(getRandom());
Response response;
try (Response esResponse = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode)) {
response = esResponse;
try {
response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode);
}
catch(ResponseException e) {
response = e.getResponse();

117
client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
View File

@ -26,9 +26,9 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpOptions;
import org.apache.http.client.methods.HttpPatch;
@ -37,11 +37,15 @@ import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpTrace;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicStatusLine;
import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
import org.apache.http.util.EntityUtils;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
@ -51,11 +55,11 @@ import org.mockito.stubbing.Answer;
import java.io.IOException;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Future;
import static org.elasticsearch.client.RestClientTestUtil.getAllErrorStatusCodes;
import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
@ -86,39 +90,48 @@ public class RestClientSingleHostTests extends RestClientTestCase {
private RestClient restClient;
private Header[] defaultHeaders;
private HttpHost httpHost;
private CloseableHttpClient httpClient;
private TrackingFailureListener failureListener;
private CloseableHttpAsyncClient httpClient;
private HostsTrackingFailureListener failureListener;
@Before
@SuppressWarnings("unchecked")
public void createRestClient() throws IOException {
httpClient = mock(CloseableHttpClient.class);
when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenAnswer(new Answer<CloseableHttpResponse>() {
@Override
public CloseableHttpResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
HttpUriRequest request = (HttpUriRequest) invocationOnMock.getArguments()[1];
//return the desired status code or exception depending on the path
if (request.getURI().getPath().equals("/soe")) {
throw new SocketTimeoutException();
} else if (request.getURI().getPath().equals("/coe")) {
throw new ConnectTimeoutException();
}
int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
httpClient = mock(CloseableHttpAsyncClient.class);
when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
@Override
public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
//return the desired status code or exception depending on the path
if (request.getURI().getPath().equals("/soe")) {
futureCallback.failed(new SocketTimeoutException());
} else if (request.getURI().getPath().equals("/coe")) {
futureCallback.failed(new ConnectTimeoutException());
} else {
int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
CloseableHttpResponse httpResponse = new CloseableBasicHttpResponse(statusLine);
//return the same body that was sent
if (request instanceof HttpEntityEnclosingRequest) {
HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
if (entity != null) {
assertTrue("the entity is not repeatable, cannot set it to the response directly", entity.isRepeatable());
httpResponse.setEntity(entity);
HttpResponse httpResponse = new BasicHttpResponse(statusLine);
//return the same body that was sent
if (request instanceof HttpEntityEnclosingRequest) {
HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
if (entity != null) {
assertTrue("the entity is not repeatable, cannot set it to the response directly",
entity.isRepeatable());
httpResponse.setEntity(entity);
}
}
//return the same headers that were sent
httpResponse.setHeaders(request.getAllHeaders());
futureCallback.completed(httpResponse);
}
return null;
}
}
//return the same headers that were sent
httpResponse.setHeaders(request.getAllHeaders());
return httpResponse;
}
});
});
int numHeaders = RandomInts.randomIntBetween(getRandom(), 0, 3);
defaultHeaders = new Header[numHeaders];
for (int i = 0; i < numHeaders; i++) {
@ -127,20 +140,22 @@ public class RestClientSingleHostTests extends RestClientTestCase {
defaultHeaders[i] = new BasicHeader(headerName, headerValue);
}
httpHost = new HttpHost("localhost", 9200);
failureListener = new TrackingFailureListener();
failureListener = new HostsTrackingFailureListener();
restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, failureListener);
}
/**
* Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client
*/
@SuppressWarnings("unchecked")
public void testInternalHttpRequest() throws Exception {
ArgumentCaptor<HttpUriRequest> requestArgumentCaptor = ArgumentCaptor.forClass(HttpUriRequest.class);
ArgumentCaptor<HttpAsyncRequestProducer> requestArgumentCaptor = ArgumentCaptor.forClass(HttpAsyncRequestProducer.class);
int times = 0;
for (String httpMethod : getHttpMethods()) {
HttpUriRequest expectedRequest = performRandomRequest(httpMethod);
verify(httpClient, times(++times)).execute(any(HttpHost.class), requestArgumentCaptor.capture());
HttpUriRequest actualRequest = requestArgumentCaptor.getValue();
verify(httpClient, times(++times)).<HttpResponse>execute(requestArgumentCaptor.capture(),
any(HttpAsyncResponseConsumer.class), any(FutureCallback.class));
HttpUriRequest actualRequest = (HttpUriRequest)requestArgumentCaptor.getValue().generateRequest();
assertEquals(expectedRequest.getURI(), actualRequest.getURI());
assertEquals(expectedRequest.getClass(), actualRequest.getClass());
assertArrayEquals(expectedRequest.getAllHeaders(), actualRequest.getAllHeaders());
@ -184,7 +199,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
/**
* End to end test for ok status codes
*/
public void testOkStatusCodes() throws Exception {
public void testOkStatusCodes() throws IOException {
for (String method : getHttpMethods()) {
for (int okStatusCode : getOkStatusCodes()) {
Response response = performRequest(method, "/" + okStatusCode);
@ -197,11 +212,12 @@ public class RestClientSingleHostTests extends RestClientTestCase {
/**
* End to end test for error status codes: they should cause an exception to be thrown, apart from 404 with HEAD requests
*/
public void testErrorStatusCodes() throws Exception {
public void testErrorStatusCodes() throws IOException {
for (String method : getHttpMethods()) {
//error status codes should cause an exception to be thrown
for (int errorStatusCode : getAllErrorStatusCodes()) {
try (Response response = performRequest(method, "/" + errorStatusCode)) {
try {
Response response = performRequest(method, "/" + errorStatusCode);
if (method.equals("HEAD") && errorStatusCode == 404) {
//no exception gets thrown although we got a 404
assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
@ -247,16 +263,14 @@ public class RestClientSingleHostTests extends RestClientTestCase {
* End to end test for request and response body. Exercises the mock http client ability to send back
* whatever body it has received.
*/
public void testBody() throws Exception {
public void testBody() throws IOException {
String body = "{ \"field\": \"value\" }";
StringEntity entity = new StringEntity(body);
for (String method : Arrays.asList("DELETE", "GET", "PATCH", "POST", "PUT")) {
for (int okStatusCode : getOkStatusCodes()) {
try (Response response = restClient.performRequest(method, "/" + okStatusCode,
Collections.<String, String>emptyMap(), entity)) {
assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
assertThat(EntityUtils.toString(response.getEntity()), equalTo(body));
}
Response response = restClient.performRequest(method, "/" + okStatusCode, Collections.<String, String>emptyMap(), entity);
assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
assertThat(EntityUtils.toString(response.getEntity()), equalTo(body));
}
for (int errorStatusCode : getAllErrorStatusCodes()) {
try {
@ -279,7 +293,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
}
}
public void testNullHeaders() throws Exception {
public void testNullHeaders() throws IOException {
String method = randomHttpMethod(getRandom());
int statusCode = randomStatusCode(getRandom());
try {
@ -296,7 +310,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
}
}
public void testNullParams() throws Exception {
public void testNullParams() throws IOException {
String method = randomHttpMethod(getRandom());
int statusCode = randomStatusCode(getRandom());
try {
@ -317,7 +331,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
* End to end test for request and response headers. Exercises the mock http client ability to send back
* whatever headers it has received.
*/
public void testHeaders() throws Exception {
public void testHeaders() throws IOException {
for (String method : getHttpMethods()) {
Map<String, String> expectedHeaders = new HashMap<>();
for (Header defaultHeader : defaultHeaders) {
@ -334,9 +348,8 @@ public class RestClientSingleHostTests extends RestClientTestCase {
int statusCode = randomStatusCode(getRandom());
Response esResponse;
try (Response response = restClient.performRequest(method, "/" + statusCode,
Collections.<String, String>emptyMap(), null, headers)) {
esResponse = response;
try {
esResponse = restClient.performRequest(method, "/" + statusCode, headers);
} catch(ResponseException e) {
esResponse = e.getResponse();
}
@ -349,7 +362,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
}
}
private HttpUriRequest performRandomRequest(String method) throws IOException, URISyntaxException {
private HttpUriRequest performRandomRequest(String method) throws Exception {
String uriAsString = "/" + randomStatusCode(getRandom());
URIBuilder uriBuilder = new URIBuilder(uriAsString);
Map<String, String> params = Collections.emptyMap();
@ -441,7 +454,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
case 1:
return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
case 2:
return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), null, headers);
return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), (HttpEntity)null, headers);
default:
throw new UnsupportedOperationException();
}

172
client/rest/src/test/java/org/elasticsearch/client/SyncResponseListenerTests.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolVersion;
import org.apache.http.RequestLine;
import org.apache.http.StatusLine;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicRequestLine;
import org.apache.http.message.BasicStatusLine;
import java.io.IOException;
import java.net.URISyntaxException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.fail;
public class SyncResponseListenerTests extends RestClientTestCase {
public void testOnSuccessNullResponse() {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
try {
syncResponseListener.onSuccess(null);
fail("onSuccess should have failed");
} catch(NullPointerException e) {
assertEquals("response must not be null", e.getMessage());
}
}
public void testOnFailureNullException() {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
try {
syncResponseListener.onFailure(null);
fail("onFailure should have failed");
} catch(NullPointerException e) {
assertEquals("exception must not be null", e.getMessage());
}
}
public void testOnSuccess() throws Exception {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
Response mockResponse = mockResponse();
syncResponseListener.onSuccess(mockResponse);
Response response = syncResponseListener.get();
assertSame(response, mockResponse);
try {
syncResponseListener.onSuccess(mockResponse);
fail("get should have failed");
} catch(IllegalStateException e) {
assertEquals(e.getMessage(), "response is already set");
}
response = syncResponseListener.get();
assertSame(response, mockResponse);
RuntimeException runtimeException = new RuntimeException("test");
syncResponseListener.onFailure(runtimeException);
try {
syncResponseListener.get();
fail("get should have failed");
} catch(IllegalStateException e) {
assertEquals("response and exception are unexpectedly set at the same time", e.getMessage());
assertNotNull(e.getSuppressed());
assertEquals(1, e.getSuppressed().length);
assertSame(runtimeException, e.getSuppressed()[0]);
}
}
public void testOnFailure() throws Exception {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
RuntimeException firstException = new RuntimeException("first-test");
syncResponseListener.onFailure(firstException);
try {
syncResponseListener.get();
fail("get should have failed");
} catch(RuntimeException e) {
assertSame(firstException, e);
}
RuntimeException secondException = new RuntimeException("second-test");
try {
syncResponseListener.onFailure(secondException);
} catch(IllegalStateException e) {
assertEquals(e.getMessage(), "exception is already set");
}
try {
syncResponseListener.get();
fail("get should have failed");
} catch(RuntimeException e) {
assertSame(firstException, e);
}
Response response = mockResponse();
syncResponseListener.onSuccess(response);
try {
syncResponseListener.get();
fail("get should have failed");
} catch(IllegalStateException e) {
assertEquals("response and exception are unexpectedly set at the same time", e.getMessage());
assertNotNull(e.getSuppressed());
assertEquals(1, e.getSuppressed().length);
assertSame(firstException, e.getSuppressed()[0]);
}
}
public void testRuntimeExceptionIsNotWrapped() throws Exception {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
RuntimeException runtimeException = new RuntimeException();
syncResponseListener.onFailure(runtimeException);
try {
syncResponseListener.get();
fail("get should have failed");
} catch(RuntimeException e) {
assertSame(runtimeException, e);
}
}
public void testIOExceptionIsNotWrapped() throws Exception {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
IOException ioException = new IOException();
syncResponseListener.onFailure(ioException);
try {
syncResponseListener.get();
fail("get should have failed");
} catch(IOException e) {
assertSame(ioException, e);
}
}
public void testExceptionIsWrapped() throws Exception {
RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000);
//we just need any checked exception
URISyntaxException exception = new URISyntaxException("test", "test");
syncResponseListener.onFailure(exception);
try {
syncResponseListener.get();
fail("get should have failed");
} catch(RuntimeException e) {
assertEquals("error while performing request", e.getMessage());
assertSame(exception, e.getCause());
}
}
private static Response mockResponse() {
ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
RequestLine requestLine = new BasicRequestLine("GET", "/", protocolVersion);
StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK");
HttpResponse httpResponse = new BasicHttpResponse(statusLine);
return new Response(requestLine, new HttpHost("localhost", 9200), httpResponse);
}
}

5
client/sniffer/build.gradle
View File

@ -58,6 +58,11 @@ forbiddenApisTest {
signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
}
dependencyLicenses {
mapping from: /http.*/, to: 'httpclient'
mapping from: /commons-.*/, to: 'commons'
}
//JarHell is part of es core, which we don't want to pull in
jarHell.enabled=false

0
client/rest/licenses/commons-logging-LICENSE.txt → client/sniffer/licenses/commons-LICENSE.txt
View File

0
client/sniffer/licenses/commons-codec-NOTICE.txt → client/sniffer/licenses/commons-NOTICE.txt
View File

6
client/sniffer/licenses/commons-logging-NOTICE.txt
View File

@ -1,6 +0,0 @@
Apache Commons Logging
Copyright 2003-2014 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

1
client/sniffer/licenses/httpcore-4.4.4.jar.sha1
View File

@ -1 +0,0 @@
b31526a230871fbe285fbcbe2813f9c0839ae9b0

1
client/sniffer/licenses/httpcore-4.4.5.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e7501a1b34325abb00d17dde96150604a0658b54

558
client/sniffer/licenses/httpcore-LICENSE.txt
View File

@ -1,558 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
=========================================================================
This project includes Public Suffix List copied from
<https://publicsuffix.org/list/effective_tld_names.dat>
licensed under the terms of the Mozilla Public License, v. 2.0
Full license text: <http://mozilla.org/MPL/2.0/>
Mozilla Public License Version 2.0
==================================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to
the creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used
by a Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached
the notice in Exhibit A, the Executable Form of such Source Code
Form, and Modifications of such Source Code Form, in each case
including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
(a) that the initial Contributor has attached the notice described
in Exhibit B to the Covered Software; or
(b) that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the
terms of a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible,
whether at the time of the initial grant or subsequently, any and
all of the rights conveyed by this License.
1.10. "Modifications"
means any of the following:
(a) any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered
Software; or
(b) any new file in Source Code Form that contains any Covered
Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the
License, by the making, using, selling, offering for sale, having
made, import, or transfer of either its Contributions or its
Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU
Lesser General Public License, Version 2.1, the GNU Affero General
Public License, Version 3.0, or any later versions of those
licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that
controls, is controlled by, or is under common control with You. For
purposes of this definition, "control" means (a) the power, direct
or indirect, to cause the direction or management of such entity,
whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial
ownership of such entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
(b) under Patent Claims of such Contributor to make, use, sell, offer
for sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
(a) for any code that a Contributor has removed from Covered Software;
or
(b) for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
(c) under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights
to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
in Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
(a) such Covered Software must also be made available in Source Code
Form, as described in Section 3.1, and You must inform recipients of
the Executable Form how they can obtain a copy of such Source Code
Form by reasonable means in a timely manner, at a charge no more
than the cost of distribution to the recipient; and
(b) You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter
the recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty,
or limitations of liability) contained within the Source Code Form of
the Covered Software, except that You may alter any license notices to
the extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Software due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description must
be placed in a text file included with all distributions of the Covered
Software under this License. Except to the extent prohibited by statute
or regulation, such description must be sufficiently detailed for a
recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically
if You fail to comply with any of its terms. However, if You become
compliant, then the rights granted under this License from a particular
Contributor are reinstated (a) provisionally, unless and until such
Contributor explicitly and finally terminates Your grants, and (b) on an
ongoing basis, if such Contributor fails to notify You of the
non-compliance by some reasonable means prior to 60 days after You have
come back into compliance. Moreover, Your grants from a particular
Contributor are reinstated on an ongoing basis if such Contributor
notifies You of the non-compliance by some reasonable means, this is the
first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after
Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
end user license agreements (excluding distributors and resellers) which
have been validly granted by You or Your distributors under this License
prior to termination shall survive termination.
************************************************************************
* *
* 6. Disclaimer of Warranty *
* ------------------------- *
* *
* Covered Software is provided under this License on an "as is" *
* basis, without warranty of any kind, either expressed, implied, or *
* statutory, including, without limitation, warranties that the *
* Covered Software is free of defects, merchantable, fit for a *
* particular purpose or non-infringing. The entire risk as to the *
* quality and performance of the Covered Software is with You. *
* Should any Covered Software prove defective in any respect, You *
* (not any Contributor) assume the cost of any necessary servicing, *
* repair, or correction. This disclaimer of warranty constitutes an *
* essential part of this License. No use of any Covered Software is *
* authorized under this License except under this disclaimer. *
* *
************************************************************************
************************************************************************
* *
* 7. Limitation of Liability *
* -------------------------- *
* *
* Under no circumstances and under no legal theory, whether tort *
* (including negligence), contract, or otherwise, shall any *
* Contributor, or anyone who distributes Covered Software as *
* permitted above, be liable to You for any direct, indirect, *
* special, incidental, or consequential damages of any character *
* including, without limitation, damages for lost profits, loss of *
* goodwill, work stoppage, computer failure or malfunction, or any *
* and all other commercial damages or losses, even if such party *
* shall have been informed of the possibility of such damages. This *
* limitation of liability shall not apply to liability for death or *
* personal injury resulting from such party's negligence to the *
* extent applicable law prohibits such limitation. Some *
* jurisdictions do not allow the exclusion or limitation of *
* incidental or consequential damages, so this exclusion and *
* limitation may not apply to You. *
* *
************************************************************************
8. Litigation
-------------
Any litigation relating to this License may be brought only in the
courts of a jurisdiction where the defendant maintains its principal
place of business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions.
Nothing in this Section shall prevent a party's ability to bring
cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides
that the language of a contract shall be construed against the drafter
shall not be used to construe this License against a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the
notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to look
for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.

6
client/sniffer/licenses/httpcore-NOTICE.txt
View File

@ -1,6 +0,0 @@
Apache HttpComponents Client
Copyright 1999-2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

5
client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
View File

@ -62,9 +62,8 @@ public class HostsSniffer {
* Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts
*/
public List<HttpHost> sniffHosts() throws IOException {
try (Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams)) {
return readHosts(response.getEntity());
}
Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams);
return readHosts(response.getEntity());
}
private List<HttpHost> readHosts(HttpEntity entity) throws IOException {

3
client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.client.sniff;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import java.io.IOException;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
@ -55,7 +54,7 @@ public class SniffOnFailureListener extends RestClient.FailureListener {
}
@Override
public void onFailure(HttpHost host) throws IOException {
public void onFailure(HttpHost host) {
if (sniffer == null) {
throw new IllegalStateException("sniffer was not set, unable to sniff on failure");
}

3
client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
View File

@ -23,6 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import java.io.Closeable;
import java.io.IOException;
@ -39,7 +40,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
* Must be created via {@link Builder}, which allows to set all of the different options or rely on defaults.
* A background task fetches the nodes through the {@link HostsSniffer} and sets them to the {@link RestClient} instance.
* It is possible to perform sniffing on failure by creating a {@link SniffOnFailureListener} and providing it as an argument to
* {@link org.elasticsearch.client.RestClient.Builder#setFailureListener(RestClient.FailureListener)}. The Sniffer implementation
* {@link RestClientBuilder#setFailureListener(RestClient.FailureListener)}. The Sniffer implementation
* needs to be lazily set to the previously created SniffOnFailureListener through {@link SniffOnFailureListener#setSniffer(Sniffer)}.
*/
public final class Sniffer implements Closeable {

3
client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
View File

@ -43,7 +43,6 @@ import java.io.OutputStream;
import java.io.StringWriter;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -86,7 +85,7 @@ public class HostsSnifferTests extends RestClientTestCase {
httpServer.stop(0);
}
public void testSniffNodes() throws IOException, URISyntaxException {
public void testSniffNodes() throws IOException {
HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort());
try (RestClient restClient = RestClient.builder(httpHost).build()) {
HostsSniffer.Builder builder = HostsSniffer.builder(restClient).setSniffRequestTimeoutMillis(sniffRequestTimeout);

17
client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
View File

@ -45,16 +45,17 @@ public class SniffOnFailureListenerTests extends RestClientTestCase {
assertEquals("sniffer must not be null", e.getMessage());
}
RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build();
try (Sniffer sniffer = Sniffer.builder(restClient, new MockHostsSniffer()).build()) {
listener.setSniffer(sniffer);
try {
try (RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build()) {
try (Sniffer sniffer = Sniffer.builder(restClient, new MockHostsSniffer()).build()) {
listener.setSniffer(sniffer);
fail("should have failed");
} catch(IllegalStateException e) {
assertEquals("sniffer can only be set once", e.getMessage());
try {
listener.setSniffer(sniffer);
fail("should have failed");
} catch(IllegalStateException e) {
assertEquals("sniffer can only be set once", e.getMessage());
}
listener.onFailure(new HttpHost("localhost", 9200));
}
listener.onFailure(new HttpHost("localhost", 9200));
}
}
}

1
client/transport/build.gradle
View File

@ -26,6 +26,7 @@ group = 'org.elasticsearch.client'
dependencies {
compile "org.elasticsearch:elasticsearch:${version}"
compile project(path: ':modules:transport-netty3', configuration: 'runtime')
compile project(path: ':modules:transport-netty4', configuration: 'runtime')
compile project(path: ':modules:reindex', configuration: 'runtime')
compile project(path: ':modules:lang-mustache', configuration: 'runtime')
compile project(path: ':modules:percolator', configuration: 'runtime')

52
client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
View File

@ -16,30 +16,47 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.reindex.ReindexPlugin;
import org.elasticsearch.percolator.PercolatorPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.mustache.MustachePlugin;
import org.elasticsearch.transport.Netty3Plugin;
import org.elasticsearch.transport.Netty4Plugin;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* A builder to create an instance of {@link TransportClient}
* This class pre-installs the {@link Netty3Plugin}, {@link ReindexPlugin}, {@link PercolatorPlugin}, and {@link MustachePlugin}
* This class pre-installs the
* {@link Netty3Plugin},
* {@link Netty4Plugin},
* {@link ReindexPlugin},
* {@link PercolatorPlugin},
* and {@link MustachePlugin}
* for the client. These plugins are all elasticsearch core modules required.
*/
@SuppressWarnings({"unchecked","varargs"})
public class PreBuiltTransportClient extends TransportClient {
private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS = Collections.unmodifiableList(Arrays.asList(
TransportPlugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class));
private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS =
Collections.unmodifiableList(
Arrays.asList(
Netty3Plugin.class,
Netty4Plugin.class,
TransportPlugin.class,
ReindexPlugin.class,
PercolatorPlugin.class,
MustachePlugin.class));
@SafeVarargs
public PreBuiltTransportClient(Settings settings, Class<? extends Plugin>... plugins) {
@ -50,14 +67,25 @@ public class PreBuiltTransportClient extends TransportClient {
super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS));
}
/**
* The default transport implementation for the transport client.
*/
public static final class TransportPlugin extends Netty3Plugin {
// disable assertions for permissions since we might not have the permissions here
// compared to if we are loaded as a real module to the es server
public TransportPlugin(Settings settings) {
super(Settings.builder().put("netty.assert.buglevel", false).put(settings).build());
public static final class TransportPlugin extends Plugin {
private static final Setting<Boolean> ASSERT_NETTY_BUGLEVEL =
Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope);
@Override
public List<Setting<?>> getSettings() {
return Collections.singletonList(ASSERT_NETTY_BUGLEVEL);
}
@Override
public Settings additionalSettings() {
return Settings.builder()
.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME)
.put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME)
.put("netty.assert.buglevel", true)
.build();
}
}
}

2
client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
View File

@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.transport.client;
import com.carrotsearch.randomizedtesting.RandomizedTest;
@ -57,4 +58,5 @@ public class PreBuiltTransportClientTests extends RandomizedTest {
}
}
}
}

25
core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
View File

@ -22,6 +22,7 @@ import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.WriteResponse;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.StatusToXContent;
@ -109,6 +110,30 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
return getShardInfo().status();
}
/**
* Gets the location of the written document as a string suitable for a {@code Location} header.
* @param routing any routing used in the request. If null the location doesn't include routing information.
*/
public String getLocation(@Nullable String routing) {
// Absolute path for the location of the document. This should be allowed as of HTTP/1.1:
// https://tools.ietf.org/html/rfc7231#section-7.1.2
String index = getIndex();
String type = getType();
String id = getId();
String routingStart = "?routing=";
int bufferSize = 3 + index.length() + type.length() + id.length();
if (routing != null) {
bufferSize += routingStart.length() + routing.length();
}
StringBuilder location = new StringBuilder(bufferSize);
location.append('/').append(index);
location.append('/').append(type);
location.append('/').append(id);
if (routing != null) {
location.append(routingStart).append(routing);
}
return location.toString();
}
@Override
public void readFrom(StreamInput in) throws IOException {

18
core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
View File

@ -142,7 +142,8 @@ final class Bootstrap {
JvmInfo.jvmInfo();
}
private void setup(boolean addShutdownHook, Settings settings, Environment environment) throws Exception {
private void setup(boolean addShutdownHook, Environment environment) throws Exception {
Settings settings = environment.settings();
initializeNatives(
environment.tmpFile(),
BootstrapSettings.MEMORY_LOCK_SETTING.get(settings),
@ -171,7 +172,7 @@ final class Bootstrap {
// install SM after natives, shutdown hooks, etc.
Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
node = new Node(settings) {
node = new Node(environment) {
@Override
protected void validateNodeBeforeAcceptingRequests(Settings settings, BoundTransportAddress boundTransportAddress) {
BootstrapCheck.check(settings, boundTransportAddress);
@ -179,7 +180,7 @@ final class Bootstrap {
};
}
private static Environment initialSettings(boolean foreground, Path pidFile, Map<String, String> esSettings) {
private static Environment initialEnvironment(boolean foreground, Path pidFile, Map<String, String> esSettings) {
Terminal terminal = foreground ? Terminal.DEFAULT : null;
Settings.Builder builder = Settings.builder();
if (pidFile != null) {
@ -225,9 +226,8 @@ final class Bootstrap {
INSTANCE = new Bootstrap();
Environment environment = initialSettings(foreground, pidFile, esSettings);
Settings settings = environment.settings();
LogConfigurator.configure(settings, true);
Environment environment = initialEnvironment(foreground, pidFile, esSettings);
LogConfigurator.configure(environment.settings(), true);
checkForCustomConfFile();
if (environment.pidFile() != null) {
@ -250,9 +250,9 @@ final class Bootstrap {
// initialized as we do not want to grant the runtime permission
// setDefaultUncaughtExceptionHandler
Thread.setDefaultUncaughtExceptionHandler(
new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(settings)));
new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(environment.settings())));
INSTANCE.setup(true, settings, environment);
INSTANCE.setup(true, environment);
INSTANCE.start();
@ -266,7 +266,7 @@ final class Bootstrap {
}
ESLogger logger = Loggers.getLogger(Bootstrap.class);
if (INSTANCE.node != null) {
logger = Loggers.getLogger(Bootstrap.class, INSTANCE.node.settings().get("node.name"));
logger = Loggers.getLogger(Bootstrap.class, Node.NODE_NAME_SETTING.get(INSTANCE.node.settings()));
}
// HACK, it sucks to do this, but we will run users out of disk space otherwise
if (e instanceof CreationException) {

21
core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
View File

@ -19,15 +19,6 @@
package org.elasticsearch.client.transport;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionListener;
@ -63,6 +54,15 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TcpTransport;
import org.elasticsearch.transport.TransportService;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* The transport client allows to create a client that is not part of the cluster, but simply connects to one
* or more nodes directly by adding their respective addresses using {@link #addTransportAddress(org.elasticsearch.common.transport.TransportAddress)}.
@ -100,6 +100,9 @@ public abstract class TransportClient extends AbstractClient {
private static ClientTemplate buildTemplate(Settings providedSettings, Settings defaultSettings,
Collection<Class<? extends Plugin>> plugins) {
if (Node.NODE_NAME_SETTING.exists(providedSettings) == false) {
providedSettings = Settings.builder().put(providedSettings).put(Node.NODE_NAME_SETTING.getKey(), "_client_").build();
}
final PluginsService pluginsService = newPluginService(providedSettings, plugins);
final Settings settings = Settings.builder().put(defaultSettings).put(pluginsService.updatedSettings()).build();
final List<Closeable> resourcesToClose = new ArrayList<>();

6
core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
View File

@ -39,7 +39,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.function.Supplier;
import static org.elasticsearch.common.transport.TransportAddressSerializers.addressToStream;
@ -191,7 +190,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
}
/** Creates a DiscoveryNode representing the local node. */
public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeIdSupplier) {
public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeId) {
Map<String, String> attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(settings).getAsMap());
Set<DiscoveryNode.Role> roles = new HashSet<>();
if (Node.NODE_INGEST_SETTING.get(settings)) {
@ -204,8 +203,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
roles.add(DiscoveryNode.Role.DATA);
}
return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeIdSupplier, publishAddress,
attributes, roles, Version.CURRENT);
return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress,attributes, roles, Version.CURRENT);
}
/**

26
core/src/main/java/org/elasticsearch/common/Strings.java
View File

@ -33,6 +33,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
@ -509,7 +510,19 @@ public class Strings {
else return s.split(",");
}
/**
* A convenience method for splitting a delimited string into
* a set and trimming leading and trailing whitespace from all
* split strings.
*
* @param s the string to split
* @param c the delimiter to split on
* @return the set of split strings
*/
public static Set<String> splitStringToSet(final String s, final char c) {
if (s == null || s.isEmpty()) {
return Collections.emptySet();
}
final char[] chars = s.toCharArray();
int count = 1;
for (final char x : chars) {
@ -521,16 +534,25 @@ public class Strings {
final int len = chars.length;
int start = 0; // starting index in chars of the current substring.
int pos = 0; // current index in chars.
int end = 0; // the position of the end of the current token
for (; pos < len; pos++) {
if (chars[pos] == c) {
int size = pos - start;
int size = end - start;
if (size > 0) { // only add non empty strings
result.add(new String(chars, start, size));
}
start = pos + 1;
end = start;
} else if (Character.isWhitespace(chars[pos])) {
if (start == pos) {
// skip over preceding whitespace
start++;
}
} else {
end = pos + 1;
}
}
int size = pos - start;
int size = end - start;
if (size > 0) {
result.add(new String(chars, start, size));
}

21
core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.common.bytes;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.ByteArray;
@ -36,24 +35,24 @@ public class PagedBytesReference extends BytesReference {
private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
private final BigArrays bigarrays;
protected final ByteArray bytearray;
protected final ByteArray byteArray;
private final int offset;
private final int length;
public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) {
this(bigarrays, bytearray, 0, length);
public PagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) {
this(bigarrays, byteArray, 0, length);
}
public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int from, int length) {
public PagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int from, int length) {
this.bigarrays = bigarrays;
this.bytearray = bytearray;
this.byteArray = byteArray;
this.offset = from;
this.length = length;
}
@Override
public byte get(int index) {
return bytearray.get(offset + index);
return byteArray.get(offset + index);
}
@Override
@ -66,14 +65,14 @@ public class PagedBytesReference extends BytesReference {
if (from < 0 || (from + length) > length()) {
throw new IllegalArgumentException("can't slice a buffer with length [" + length() + "], with slice parameters from [" + from + "], length [" + length + "]");
}
return new PagedBytesReference(bigarrays, bytearray, offset + from, length);
return new PagedBytesReference(bigarrays, byteArray, offset + from, length);
}
@Override
public BytesRef toBytesRef() {
BytesRef bref = new BytesRef();
// if length <= pagesize this will dereference the page, or materialize the byte[]
bytearray.get(offset, length, bref);
byteArray.get(offset, length, bref);
return bref;
}
@ -95,7 +94,7 @@ public class PagedBytesReference extends BytesReference {
@Override
public BytesRef next() throws IOException {
if (nextFragmentSize != 0) {
final boolean materialized = bytearray.get(offset + position, nextFragmentSize, slice);
final boolean materialized = byteArray.get(offset + position, nextFragmentSize, slice);
assert materialized == false : "iteration should be page aligned but array got materialized";
position += nextFragmentSize;
final int remaining = length - position;
@ -111,6 +110,6 @@ public class PagedBytesReference extends BytesReference {
@Override
public long ramBytesUsed() {
return bytearray.ramBytesUsed();
return byteArray.ramBytesUsed();
}
}

7
core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
View File

@ -30,12 +30,13 @@ import org.elasticsearch.common.util.ByteArray;
*/
public final class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable {
public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) {
super(bigarrays, bytearray, length);
public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) {
super(bigarrays, byteArray, length);
}
@Override
public void close() {
Releasables.close(bytearray);
Releasables.close(byteArray);
}
}

3
core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
/**
*
@ -50,7 +51,7 @@ public abstract class AbstractComponent {
* Returns the nodes name from the settings or the empty string if not set.
*/
public final String nodeName() {
return settings.get("node.name", "");
return Node.NODE_NAME_SETTING.get(settings);
}
/**

1
core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java
View File

@ -28,4 +28,5 @@ public interface ReleasableBytesStream extends BytesStream {
@Override
ReleasablePagedBytesReference bytes();
}

14
core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java
View File

@ -33,7 +33,7 @@ import java.io.IOException;
*/
public class BytesStreamOutput extends StreamOutput implements BytesStream {
protected final BigArrays bigarrays;
protected final BigArrays bigArrays;
protected ByteArray bytes;
protected int count;
@ -57,9 +57,9 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
this(expectedSize, BigArrays.NON_RECYCLING_INSTANCE);
}
protected BytesStreamOutput(int expectedSize, BigArrays bigarrays) {
this.bigarrays = bigarrays;
this.bytes = bigarrays.newByteArray(expectedSize);
protected BytesStreamOutput(int expectedSize, BigArrays bigArrays) {
this.bigArrays = bigArrays;
this.bytes = bigArrays.newByteArray(expectedSize);
}
@Override
@ -100,7 +100,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
public void reset() {
// shrink list of pages
if (bytes.size() > BigArrays.PAGE_SIZE_IN_BYTES) {
bytes = bigarrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES);
bytes = bigArrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES);
}
// go back to start
@ -145,7 +145,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
@Override
public BytesReference bytes() {
return new PagedBytesReference(bigarrays, bytes, count);
return new PagedBytesReference(bigArrays, bytes, count);
}
/**
@ -157,7 +157,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
}
private void ensureCapacity(int offset) {
bytes = bigarrays.grow(bytes, offset);
bytes = bigArrays.grow(bytes, offset);
}
}

7
core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java
View File

@ -36,12 +36,13 @@ public class ReleasableBytesStreamOutput extends BytesStreamOutput implements Re
super(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays);
}
public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigarrays) {
super(expectedSize, bigarrays);
public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigArrays) {
super(expectedSize, bigArrays);
}
@Override
public ReleasablePagedBytesReference bytes() {
return new ReleasablePagedBytesReference(bigarrays, bytes, count);
return new ReleasablePagedBytesReference(bigArrays, bytes, count);
}
}

1
core/src/main/java/org/elasticsearch/common/lease/Releasable.java
View File

@ -30,4 +30,5 @@ public interface Releasable extends Closeable {
@Override
void close();
}

6
core/src/main/java/org/elasticsearch/common/logging/Loggers.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.node.Node;
import java.net.InetAddress;
import java.net.UnknownHostException;
@ -101,9 +102,8 @@ public class Loggers {
prefixesList.add(addr.getHostName());
}
}
String name = settings.get("node.name");
if (name != null) {
prefixesList.add(name);
if (Node.NODE_NAME_SETTING.exists(settings)) {
prefixesList.add(Node.NODE_NAME_SETTING.get(settings));
}
if (prefixes != null && prefixes.length > 0) {
prefixesList.addAll(asList(prefixes));

3
core/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java
View File

@ -55,6 +55,9 @@ public class NetworkExceptionHelper {
if (e.getMessage().contains("Connection timed out")) {
return true;
}
if (e.getMessage().equals("Socket is closed")) {
return true;
}
}
return false;
}

4
core/src/main/java/org/elasticsearch/common/util/BigArrays.java
View File

@ -429,6 +429,10 @@ public class BigArrays implements Releasable {
return this.circuitBreakingInstance;
}
public CircuitBreakerService breakerService() {
return this.circuitBreakingInstance.breakerService;
}
private <T extends AbstractBigArray> T resizeInPlace(T array, long newSize) {
final long oldMemSize = array.ramBytesUsed();
array.resize(newSize);

8
core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.util.concurrent;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.Node;
import java.util.Arrays;
import java.util.concurrent.BlockingQueue;
@ -83,11 +84,10 @@ public class EsExecutors {
}
public static String threadName(Settings settings, String namePrefix) {
String nodeName = settings.get("node.name");
if (nodeName == null) {
return threadName("", namePrefix);
if (Node.NODE_NAME_SETTING.exists(settings)) {
return threadName(Node.NODE_NAME_SETTING.get(settings), namePrefix);
} else {
return threadName(nodeName, namePrefix);
return threadName("", namePrefix);
}
}

3
core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java
View File

@ -21,7 +21,8 @@ package org.elasticsearch.common.xcontent;
import org.elasticsearch.rest.RestStatus;
/**
*
* Objects that can both render themselves in as json/yaml/etc and can provide a {@link RestStatus} for their response. Usually should be
* implemented by top level responses sent back to users from REST endpoints.
*/
public interface StatusToXContent extends ToXContent {

31
core/src/main/java/org/elasticsearch/env/Environment.java vendored
View File

@ -36,6 +36,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import static org.elasticsearch.common.Strings.cleanPath;
@ -107,7 +108,6 @@ public class Environment {
}
public Environment(Settings settings) {
this.settings = settings;
final Path homeFile;
if (PATH_HOME_SETTING.exists(settings)) {
homeFile = PathUtils.get(cleanPath(PATH_HOME_SETTING.get(settings)));
@ -171,6 +171,13 @@ public class Environment {
binFile = homeFile.resolve("bin");
libFile = homeFile.resolve("lib");
modulesFile = homeFile.resolve("modules");
Settings.Builder finalSettings = Settings.builder().put(settings);
finalSettings.put(PATH_HOME_SETTING.getKey(), homeFile);
finalSettings.putArray(PATH_DATA_SETTING.getKey(), dataPaths);
finalSettings.put(PATH_LOGS_SETTING.getKey(), logsFile);
this.settings = finalSettings.build();
}
/**
@ -332,4 +339,26 @@ public class Environment {
public static FileStore getFileStore(Path path) throws IOException {
return ESFileStore.getMatchingFileStore(path, fileStores);
}
/**
* asserts that the two environments are equivalent for all things the environment cares about (i.e., all but the setting
* object which may contain different setting)
*/
public static void assertEquivalent(Environment actual, Environment expected) {
assertEquals(actual.dataWithClusterFiles(), expected.dataWithClusterFiles(), "dataWithClusterFiles");
assertEquals(actual.repoFiles(), expected.repoFiles(), "repoFiles");
assertEquals(actual.configFile(), expected.configFile(), "configFile");
assertEquals(actual.scriptsFile(), expected.scriptsFile(), "scriptsFile");
assertEquals(actual.pluginsFile(), expected.pluginsFile(), "pluginsFile");
assertEquals(actual.binFile(), expected.binFile(), "binFile");
assertEquals(actual.libFile(), expected.libFile(), "libFile");
assertEquals(actual.modulesFile(), expected.modulesFile(), "modulesFile");
assertEquals(actual.logsFile(), expected.logsFile(), "logsFile");
assertEquals(actual.pidFile(), expected.pidFile(), "pidFile");
assertEquals(actual.tmpFile(), expected.tmpFile(), "tmpFile");
}
private static void assertEquals(Object actual, Object expected, String name) {
assert Objects.deepEquals(actual, expected) : "actual " + name + " [" + actual + "] is different than [ " + expected + "]";
}
}

35
core/src/main/java/org/elasticsearch/env/NodeEnvironment.java vendored
View File

@ -34,10 +34,10 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
@ -52,6 +52,7 @@ import org.elasticsearch.index.store.FsDirectoryService;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.monitor.fs.FsProbe;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.node.Node;
import java.io.Closeable;
import java.io.IOException;
@ -79,7 +80,9 @@ import static java.util.Collections.unmodifiableSet;
/**
* A component that holds all data paths for a single node.
*/
public final class NodeEnvironment extends AbstractComponent implements Closeable {
public final class NodeEnvironment implements Closeable {
private final ESLogger logger;
public static class NodePath {
/* ${data.paths}/nodes/{node.id} */
@ -139,8 +142,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
private final Path sharedDataPath;
private final Lock[] locks;
private final boolean addLockIdToCustomPath;
private final int nodeLockId;
private final AtomicBoolean closed = new AtomicBoolean(false);
private final Map<ShardId, InternalShardLock> shardLocks = new HashMap<>();
@ -177,12 +178,8 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
public static final String NODES_FOLDER = "nodes";
public static final String INDICES_FOLDER = "indices";
public static final String NODE_LOCK_FILENAME = "node.lock";
public static final String UPGRADE_LOCK_FILENAME = "upgrade.lock";
public NodeEnvironment(Settings settings, Environment environment) throws IOException {
super(settings);
this.addLockIdToCustomPath = ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.get(settings);
if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) {
nodePaths = null;
@ -190,12 +187,16 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
locks = null;
nodeLockId = -1;
nodeMetaData = new NodeMetaData(generateNodeId(settings));
logger = Loggers.getLogger(getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId()));
return;
}
final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
final Lock[] locks = new Lock[nodePaths.length];
boolean success = false;
// trace logger to debug issues before the default node name is derived from the node id
ESLogger startupTraceLogger = Loggers.getLogger(getClass(), settings);
try {
sharedDataPath = environment.sharedDataFile();
int nodeLockId = -1;
@ -207,7 +208,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
Path dataDir = environment.dataFiles()[dirIndex];
// TODO: Remove this in 6.0, we are no longer going to read from the cluster name directory
if (readFromDataPathWithClusterName(dataDirWithClusterName)) {
DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
DeprecationLogger deprecationLogger = new DeprecationLogger(startupTraceLogger);
deprecationLogger.deprecated("ES has detected the [path.data] folder using the cluster name as a folder [{}], " +
"Elasticsearch 6.0 will not allow the cluster name as a folder within the data path", dataDir);
dataDir = dataDirWithClusterName;
@ -216,20 +217,20 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
Files.createDirectories(dir);
try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
startupTraceLogger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
try {
locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
nodePaths[dirIndex] = new NodePath(dir);
nodeLockId = possibleLockId;
} catch (LockObtainFailedException ex) {
logger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
startupTraceLogger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
// release all the ones that were obtained up until now
releaseAndNullLocks(locks);
break;
}
} catch (IOException e) {
logger.trace("failed to obtain node lock on {}", e, dir.toAbsolutePath());
startupTraceLogger.trace("failed to obtain node lock on {}", e, dir.toAbsolutePath());
lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e);
// release all the ones that were obtained up until now
releaseAndNullLocks(locks);
@ -246,6 +247,8 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
throw new IllegalStateException("Failed to obtain node lock, is the following location writable?: "
+ Arrays.toString(environment.dataWithClusterFiles()), lastException);
}
this.nodeMetaData = loadOrCreateNodeMetaData(settings, startupTraceLogger, nodePaths);
this.logger = Loggers.getLogger(getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId()));
this.nodeLockId = nodeLockId;
this.locks = locks;
@ -258,8 +261,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
maybeLogPathDetails();
maybeLogHeapDetails();
this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
applySegmentInfosTrace(settings);
assertCanWrite();
success = true;
@ -924,10 +925,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
}
}
Settings getSettings() { // for testing
return settings;
}
/**
* Resolve the custom path for a index's shard.
* Uses the {@code IndexMetaData.SETTING_DATA_PATH} setting to determine
@ -940,7 +937,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
if (customDataDir != null) {
// This assert is because this should be caught by MetaDataCreateIndexService
assert sharedDataPath != null;
if (addLockIdToCustomPath) {
if (ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.get(indexSettings.getNodeSettings())) {
return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.nodeLockId));
} else {
return sharedDataPath.resolve(customDataDir);

1
core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java
View File

@ -29,4 +29,5 @@ import org.elasticsearch.rest.RestRequest;
public interface HttpServerAdapter {
void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext context);
}

4
core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
View File

@ -22,9 +22,6 @@ package org.elasticsearch.http;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.transport.BoundTransportAddress;
/**
*
*/
public interface HttpServerTransport extends LifecycleComponent {
BoundTransportAddress boundAddress();
@ -34,4 +31,5 @@ public interface HttpServerTransport extends LifecycleComponent {
HttpStats stats();
void httpServerAdapter(HttpServerAdapter httpServerAdapter);
}

4
core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
View File

@ -40,9 +40,9 @@ public final class HttpTransportSettings {
public static final Setting<Integer> SETTING_CORS_MAX_AGE =
Setting.intSetting("http.cors.max-age", 1728000, Property.NodeScope);
public static final Setting<String> SETTING_CORS_ALLOW_METHODS =
new Setting<>("http.cors.allow-methods", "OPTIONS, HEAD, GET, POST, PUT, DELETE", (value) -> value, Property.NodeScope);
new Setting<>("http.cors.allow-methods", "OPTIONS,HEAD,GET,POST,PUT,DELETE", (value) -> value, Property.NodeScope);
public static final Setting<String> SETTING_CORS_ALLOW_HEADERS =
new Setting<>("http.cors.allow-headers", "X-Requested-With, Content-Type, Content-Length", (value) -> value, Property.NodeScope);
new Setting<>("http.cors.allow-headers", "X-Requested-With,Content-Type,Content-Length", (value) -> value, Property.NodeScope);
public static final Setting<Boolean> SETTING_CORS_ALLOW_CREDENTIALS =
Setting.boolSetting("http.cors.allow-credentials", false, Property.NodeScope);
public static final Setting<Boolean> SETTING_PIPELINING =

3
core/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.node.Node;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
@ -227,7 +228,7 @@ public final class IndexSettings {
this.index = indexMetaData.getIndex();
version = Version.indexCreated(settings);
logger = Loggers.getLogger(getClass(), settings, index);
nodeName = settings.get("node.name", "");
nodeName = Node.NODE_NAME_SETTING.get(settings);
this.indexMetaData = indexMetaData;
numberOfShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
isShadowReplicaIndex = IndexMetaData.isIndexUsingShadowReplicas(settings);

12
core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
View File

@ -34,8 +34,8 @@ import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext;
import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.docvalues.DocValueFieldsContext;
import org.elasticsearch.search.fetch.docvalues.DocValueFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
import org.elasticsearch.search.highlight.HighlightBuilder;
@ -574,12 +574,12 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
}
}
if (docValueFields != null) {
FieldDataFieldsContext fieldDataFieldsContext = innerHitsContext
.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
DocValueFieldsContext docValueFieldsContext = innerHitsContext
.getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY);
for (String field : docValueFields) {
fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(field));
docValueFieldsContext.add(new DocValueFieldsContext.DocValueField(field));
}
fieldDataFieldsContext.setHitExecutionNeeded(true);
docValueFieldsContext.setHitExecutionNeeded(true);
}
if (scriptFields != null) {
for (ScriptField field : scriptFields) {

14
core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
View File

@ -57,7 +57,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
new Setting<>("indices.breaker.fielddata.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
public static final Setting<ByteSizeValue> REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING =
Setting.byteSizeSetting("indices.breaker.request.limit", "40%", Property.Dynamic, Property.NodeScope);
Setting.byteSizeSetting("indices.breaker.request.limit", "60%", Property.Dynamic, Property.NodeScope);
public static final Setting<Double> REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING =
Setting.doubleSetting("indices.breaker.request.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope);
public static final Setting<CircuitBreaker.Type> REQUEST_CIRCUIT_BREAKER_TYPE_SETTING =
@ -98,7 +98,10 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.get(settings)
);
this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT, TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(), 1.0, CircuitBreaker.Type.PARENT);
this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT,
TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(), 1.0,
CircuitBreaker.Type.PARENT);
if (logger.isTraceEnabled()) {
logger.trace("parent circuit breaker with settings {}", this.parentSettings);
}
@ -137,7 +140,6 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
registerBreaker(newFielddataSettings);
HierarchyCircuitBreakerService.this.fielddataSettings = newFielddataSettings;
logger.info("Updated breaker settings field data: {}", newFielddataSettings);
}
private boolean validateTotalCircuitBreakerLimit(ByteSizeValue byteSizeValue) {
@ -184,7 +186,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
}
// Manually add the parent breaker settings since they aren't part of the breaker map
allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(),
parentEstimated, 1.0, parentTripCount.get()));
parentEstimated, 1.0, parentTripCount.get()));
return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()]));
}
@ -207,8 +209,8 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
if (totalUsed > parentLimit) {
this.parentTripCount.incrementAndGet();
throw new CircuitBreakingException("[parent] Data too large, data for [" +
label + "] would be larger than limit of [" +
parentLimit + "/" + new ByteSizeValue(parentLimit) + "]",
label + "] would be larger than limit of [" +
parentLimit + "/" + new ByteSizeValue(parentLimit) + "]",
totalUsed, parentLimit);
}
}

1
core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
View File

@ -109,6 +109,7 @@ public class CompoundProcessor implements Processor {
throw compoundProcessorException;
} else {
executeOnFailure(ingestDocument, compoundProcessorException);
break;
}
}
}

155
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -132,7 +132,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
@ -170,7 +169,16 @@ public class Node implements Closeable {
}
}, Setting.Property.NodeScope);
/**
* Adds a default node name to the given setting, if it doesn't already exist
* @return the given setting if node name is already set, or a new copy with a default node name set.
*/
public static final Settings addNodeNameIfNeeded(Settings settings, final String nodeId) {
if (NODE_NAME_SETTING.exists(settings)) {
return settings;
}
return Settings.builder().put(settings).put(NODE_NAME_SETTING.getKey(), nodeId.substring(0, 7)).build();
}
private static final String CLIENT_TYPE = "node";
private final Lifecycle lifecycle = new Lifecycle();
@ -188,53 +196,82 @@ public class Node implements Closeable {
* @param preparedSettings Base settings to configure the node with
*/
public Node(Settings preparedSettings) {
this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), Collections.<Class<? extends Plugin>>emptyList());
this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null));
}
protected Node(Environment tmpEnv, Collection<Class<? extends Plugin>> classpathPlugins) {
Settings tmpSettings = Settings.builder().put(tmpEnv.settings())
.put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build();
public Node(Environment environment) {
this(environment, Collections.emptyList());
}
protected Node(final Environment environment, Collection<Class<? extends Plugin>> classpathPlugins) {
final List<Closeable> resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error
tmpSettings = TribeService.processSettings(tmpSettings);
ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(tmpSettings));
final String displayVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
final JvmInfo jvmInfo = JvmInfo.jvmInfo();
logger.info(
"version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]",
displayVersion,
jvmInfo.pid(),
Build.CURRENT.shortHash(),
Build.CURRENT.date(),
Constants.OS_NAME,
Constants.OS_VERSION,
Constants.OS_ARCH,
Constants.JVM_VENDOR,
Constants.JVM_NAME,
Constants.JAVA_VERSION,
Constants.JVM_VERSION);
logger.info("initializing ...");
if (logger.isDebugEnabled()) {
logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]",
tmpEnv.configFile(), Arrays.toString(tmpEnv.dataFiles()), tmpEnv.logsFile(), tmpEnv.pluginsFile());
}
// TODO: Remove this in Elasticsearch 6.0.0
if (JsonXContent.unquotedFieldNamesSet) {
DeprecationLogger dLogger = new DeprecationLogger(logger);
dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0",
JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES);
}
this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins);
this.settings = pluginsService.updatedSettings();
// create the environment based on the finalized (processed) view of the settings
this.environment = new Environment(this.settings);
final List<ExecutorBuilder<?>> executorBuilders = pluginsService.getExecutorBuilders(settings);
boolean success = false;
{
// use temp logger just to say we are starting. we can't use it later on because the node name might not be set
ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(environment.settings()));
logger.info("initializing ...");
}
try {
Settings tmpSettings = Settings.builder().put(environment.settings())
.put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build();
tmpSettings = TribeService.processSettings(tmpSettings);
// create the node environment as soon as possible, to recover the node id and enable logging
try {
nodeEnvironment = new NodeEnvironment(tmpSettings, environment);
resourcesToClose.add(nodeEnvironment);
} catch (IOException ex) {
throw new IllegalStateException("Failed to created node environment", ex);
}
final boolean hadPredefinedNodeName = NODE_NAME_SETTING.exists(tmpSettings);
tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeEnvironment.nodeId());
ESLogger logger = Loggers.getLogger(Node.class, tmpSettings);
if (hadPredefinedNodeName == false) {
logger.info("node name [{}] derived from node ID; set [{}] to override",
NODE_NAME_SETTING.get(tmpSettings), NODE_NAME_SETTING.getKey());
}
final String displayVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
final JvmInfo jvmInfo = JvmInfo.jvmInfo();
logger.info(
"version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]",
displayVersion,
jvmInfo.pid(),
Build.CURRENT.shortHash(),
Build.CURRENT.date(),
Constants.OS_NAME,
Constants.OS_VERSION,
Constants.OS_ARCH,
Constants.JVM_VENDOR,
Constants.JVM_NAME,
Constants.JAVA_VERSION,
Constants.JVM_VERSION);
if (logger.isDebugEnabled()) {
logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]",
environment.configFile(), Arrays.toString(environment.dataFiles()), environment.logsFile(), environment.pluginsFile());
}
// TODO: Remove this in Elasticsearch 6.0.0
if (JsonXContent.unquotedFieldNamesSet) {
DeprecationLogger dLogger = new DeprecationLogger(logger);
dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0",
JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES);
}
this.pluginsService = new PluginsService(tmpSettings, environment.modulesFile(), environment.pluginsFile(), classpathPlugins);
this.settings = pluginsService.updatedSettings();
// create the environment based on the finalized (processed) view of the settings
// this is just to makes sure that people get the same settings, no matter where they ask them from
this.environment = new Environment(this.settings);
Environment.assertEquivalent(environment, this.environment);
final List<ExecutorBuilder<?>> executorBuilders = pluginsService.getExecutorBuilders(settings);
final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0]));
resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
// adds the context to the DeprecationLogger so that it does not need to be injected everywhere
@ -249,19 +286,13 @@ public class Node implements Closeable {
additionalSettings.addAll(builder.getRegisteredSettings());
}
final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
final ScriptModule scriptModule = ScriptModule.create(settings, environment, resourceWatcherService,
pluginsService.filterPlugins(ScriptPlugin.class));
AnalysisModule analysisModule = new AnalysisModule(environment, pluginsService.filterPlugins(AnalysisPlugin.class));
final ScriptModule scriptModule = ScriptModule.create(settings, this.environment, resourceWatcherService,
pluginsService.filterPlugins(ScriptPlugin.class));
AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class));
additionalSettings.addAll(scriptModule.getSettings());
// this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
// so we might be late here already
final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter);
try {
nodeEnvironment = new NodeEnvironment(this.settings, this.environment);
resourcesToClose.add(nodeEnvironment);
} catch (IOException ex) {
throw new IllegalStateException("Failed to created node environment", ex);
}
resourcesToClose.add(resourceWatcherService);
final NetworkService networkService = new NetworkService(settings);
final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
@ -270,7 +301,7 @@ public class Node implements Closeable {
final TribeService tribeService = new TribeService(settings, clusterService, nodeEnvironment.nodeId());
resourcesToClose.add(tribeService);
NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
final IngestService ingestService = new IngestService(settings, threadPool, environment,
final IngestService ingestService = new IngestService(settings, threadPool, this.environment,
scriptModule.getScriptService(), pluginsService.filterPlugins(IngestPlugin.class));
ModulesBuilder modules = new ModulesBuilder();
@ -287,10 +318,10 @@ public class Node implements Closeable {
modules.add(new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class)));
modules.add(new SearchModule(settings, namedWriteableRegistry, false, pluginsService.filterPlugins(SearchPlugin.class)));
modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false, settings,
clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(),
pluginsService.filterPlugins(ActionPlugin.class)));
clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(),
pluginsService.filterPlugins(ActionPlugin.class)));
modules.add(new GatewayModule());
modules.add(new RepositoriesModule(environment, pluginsService.filterPlugins(RepositoryPlugin.class)));
modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class)));
pluginsService.processModules(modules);
CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
settingsModule.getClusterSettings());
@ -306,7 +337,7 @@ public class Node implements Closeable {
b.bind(PluginsService.class).toInstance(pluginsService);
b.bind(Client.class).toInstance(client);
b.bind(NodeClient.class).toInstance(client);
b.bind(Environment.class).toInstance(environment);
b.bind(Environment.class).toInstance(this.environment);
b.bind(ThreadPool.class).toInstance(threadPool);
b.bind(NodeEnvironment.class).toInstance(nodeEnvironment);
b.bind(TribeService.class).toInstance(tribeService);
@ -316,14 +347,14 @@ public class Node implements Closeable {
b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry());
b.bind(IngestService.class).toInstance(ingestService);
pluginComponents.stream().forEach(p -> b.bind((Class)p.getClass()).toInstance(p));
pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p));
}
);
injector = modules.createInjector();
List<LifecycleComponent> pluginLifecycleComponents = pluginComponents.stream()
.filter(p -> p instanceof LifecycleComponent)
.map(p -> (LifecycleComponent)p).collect(Collectors.toList());
.map(p -> (LifecycleComponent) p).collect(Collectors.toList());
pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream()
.map(injector::getInstance).collect(Collectors.toList()));
resourcesToClose.addAll(pluginLifecycleComponents);
@ -331,6 +362,8 @@ public class Node implements Closeable {
client.intialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}));
logger.info("initialized");
success = true;
} catch (IOException ex) {
throw new ElasticsearchException("failed to bind service", ex);
@ -339,8 +372,6 @@ public class Node implements Closeable {
IOUtils.closeWhileHandlingException(resourcesToClose);
}
}
logger.info("initialized");
}
/**

48
core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
View File

@ -21,18 +21,13 @@ package org.elasticsearch.node.internal;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.env.Environment;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@ -64,7 +59,7 @@ public class InternalSettingsPreparer {
public static Settings prepareSettings(Settings input) {
Settings.Builder output = Settings.builder();
initializeSettings(output, input, true, Collections.emptyMap());
finalizeSettings(output, null, null);
finalizeSettings(output, null);
return output.build();
}
@ -120,7 +115,7 @@ public class InternalSettingsPreparer {
// re-initialize settings now that the config file has been loaded
// TODO: only re-initialize if a config file was actually loaded
initializeSettings(output, input, false, properties);
finalizeSettings(output, terminal, environment.configFile());
finalizeSettings(output, terminal);
environment = new Environment(output.build());
@ -145,9 +140,8 @@ public class InternalSettingsPreparer {
/**
* Finish preparing settings by replacing forced settings, prompts, and any defaults that need to be added.
* The provided terminal is used to prompt for settings needing to be replaced.
* The provided configDir is optional and will be used to lookup names.txt if the node name is not set, if provided.
*/
private static void finalizeSettings(Settings.Builder output, Terminal terminal, Path configDir) {
private static void finalizeSettings(Settings.Builder output, Terminal terminal) {
// allow to force set properties based on configuration of the settings provided
List<String> forcedSettings = new ArrayList<>();
for (String setting : output.internalMap().keySet()) {
@ -167,42 +161,6 @@ public class InternalSettingsPreparer {
}
replacePromptPlaceholders(output, terminal);
// all settings placeholders have been resolved. resolve the value for the name setting by checking for name,
// then looking for node.name, and finally generate one if needed
String name = output.get("node.name");
if (name == null || name.isEmpty()) {
name = randomNodeName(configDir);
output.put("node.name", name);
}
}
private static String randomNodeName(Path configDir) {
InputStream input;
if (configDir != null && Files.exists(configDir.resolve("names.txt"))) {
Path namesPath = configDir.resolve("names.txt");
try {
input = Files.newInputStream(namesPath);
} catch (IOException e) {
throw new RuntimeException("Failed to load custom names.txt from " + namesPath, e);
}
} else {
input = InternalSettingsPreparer.class.getResourceAsStream("/config/names.txt");
}
try {
List<String> names = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) {
String name = reader.readLine();
while (name != null) {
names.add(name);
name = reader.readLine();
}
}
int index = Randomness.get().nextInt(names.size());
return names.get(index);
} catch (IOException e) {
throw new RuntimeException("Could not read node names list", e);
}
}
private static void replacePromptPlaceholders(Settings.Builder settings, Terminal terminal) {

1
core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
View File

@ -101,4 +101,5 @@ public abstract class AbstractRestChannel implements RestChannel {
public boolean detailedErrorsEnabled() {
return detailedErrorsEnabled;
}
}

2
core/src/main/java/org/elasticsearch/rest/RestChannel.java
View File

@ -30,6 +30,7 @@ import java.io.IOException;
* A channel used to construct bytes / builder based outputs, and send responses.
*/
public interface RestChannel {
XContentBuilder newBuilder() throws IOException;
XContentBuilder newErrorBuilder() throws IOException;
@ -46,4 +47,5 @@ public interface RestChannel {
boolean detailedErrorsEnabled();
void sendResponse(RestResponse response);
}

2
core/src/main/java/org/elasticsearch/rest/RestController.java
View File

@ -193,7 +193,7 @@ public class RestController extends AbstractLifecycleComponent {
if (!checkRequestParameters(request, channel)) {
return;
}
try (ThreadContext.StoredContext t = threadContext.stashContext()) {
try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
for (String key : headersToCopy) {
String httpHeader = request.header(key);
if (httpHeader != null) {

50
core/src/main/java/org/elasticsearch/rest/RestRequest.java
View File

@ -29,16 +29,35 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.rest.support.RestUtils;
import java.net.SocketAddress;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue;
import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
/**
*
*/
public abstract class RestRequest implements ToXContent.Params {
private final Map<String, String> params;
private final String rawPath;
public RestRequest(String uri) {
final Map<String, String> params = new HashMap<>();
int pathEndPos = uri.indexOf('?');
if (pathEndPos < 0) {
this.rawPath = uri;
} else {
this.rawPath = uri.substring(0, pathEndPos);
RestUtils.decodeQueryString(uri, pathEndPos + 1, params);
}
this.params = params;
}
public RestRequest(Map<String, String> params, String path) {
this.params = params;
this.rawPath = path;
}
public enum Method {
GET, POST, PUT, DELETE, OPTIONS, HEAD
}
@ -53,7 +72,9 @@ public abstract class RestRequest implements ToXContent.Params {
/**
* The non decoded, raw path provided.
*/
public abstract String rawPath();
public String rawPath() {
return rawPath;
}
/**
* The path part of the URI (without the query string), decoded.
@ -80,12 +101,27 @@ public abstract class RestRequest implements ToXContent.Params {
return null;
}
public abstract boolean hasParam(String key);
public final boolean hasParam(String key) {
return params.containsKey(key);
}
@Override
public abstract String param(String key);
public final String param(String key) {
return params.get(key);
}
public abstract Map<String, String> params();
@Override
public final String param(String key, String defaultValue) {
String value = params.get(key);
if (value == null) {
return defaultValue;
}
return value;
}
public Map<String, String> params() {
return params;
}
public float paramAsFloat(String key, float defaultValue) {
String sValue = param(key);

5
core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
View File

@ -90,7 +90,8 @@ public class RestIndexAction extends BaseRestHandler {
} catch (IllegalArgumentException eia){
try {
XContentBuilder builder = channel.newErrorBuilder();
channel.sendResponse(new BytesRestResponse(BAD_REQUEST, builder.startObject().field("error", eia.getMessage()).endObject()));
channel.sendResponse(
new BytesRestResponse(BAD_REQUEST, builder.startObject().field("error", eia.getMessage()).endObject()));
} catch (IOException e1) {
logger.warn("Failed to send response", e1);
return;
@ -101,6 +102,6 @@ public class RestIndexAction extends BaseRestHandler {
if (consistencyLevel != null) {
indexRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
}
client.index(indexRequest, new RestStatusToXContentListener<>(channel));
client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing())));
}
}

28
core/src/main/java/org/elasticsearch/rest/action/support/RestStatusToXContentListener.java
View File

@ -23,14 +23,30 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import java.util.function.Function;
/**
*
* Content listener that extracts that {@link RestStatus} from the response.
*/
public class RestStatusToXContentListener<Response extends StatusToXContent> extends RestResponseListener<Response> {
private final Function<Response, String> extractLocation;
/**
* Build an instance that doesn't support responses with the status {@code 201 CREATED}.
*/
public RestStatusToXContentListener(RestChannel channel) {
// TODO switch this to throwing an exception?
this(channel, r -> null);
}
/**
* Build an instance that does support responses with the status {@code 201 CREATED}.
*/
public RestStatusToXContentListener(RestChannel channel, Function<Response, String> extractLocation) {
super(channel);
this.extractLocation = extractLocation;
}
@Override
@ -42,7 +58,13 @@ public class RestStatusToXContentListener<Response extends StatusToXContent> ext
builder.startObject();
response.toXContent(builder, channel.request());
builder.endObject();
return new BytesRestResponse(response.status(), builder);
BytesRestResponse restResponse = new BytesRestResponse(response.status(), builder);
if (RestStatus.CREATED == restResponse.status()) {
String location = extractLocation.apply(response);
if (location != null) {
restResponse.addHeader("Location", location);
}
}
return restResponse;
}
}

2
core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
View File

@ -97,6 +97,6 @@ public class RestUpdateAction extends BaseRestHandler {
}
}
client.update(updateRequest, new RestStatusToXContentListener<>(channel));
client.update(updateRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(updateRequest.routing())));
}
}

4
core/src/main/java/org/elasticsearch/search/SearchModule.java
View File

@ -246,7 +246,7 @@ import org.elasticsearch.search.controller.SearchPhaseController;
import org.elasticsearch.search.fetch.FetchPhase;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.docvalues.DocValueFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase;
import org.elasticsearch.search.fetch.parent.ParentFieldSubFetchPhase;
import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
@ -792,7 +792,7 @@ public class SearchModule extends AbstractModule {
private void registerFetchSubPhases(List<SearchPlugin> plugins) {
registerFetchSubPhase(new ExplainFetchSubPhase());
registerFetchSubPhase(new FieldDataFieldsFetchSubPhase());
registerFetchSubPhase(new DocValueFieldsFetchSubPhase());
registerFetchSubPhase(new ScriptFieldsFetchSubPhase());
registerFetchSubPhase(new FetchSourceSubPhase());
registerFetchSubPhase(new VersionFetchSubPhase());

Some files were not shown because too many files have changed in this diff Show More