diff --git a/build.gradle b/build.gradle index 807e3cfd222..c2bf6c9309f 100644 --- a/build.gradle +++ b/build.gradle @@ -175,6 +175,7 @@ subprojects { "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', // for transport client "org.elasticsearch.plugin:transport-netty3-client:${version}": ':modules:transport-netty3', + "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4', "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex', "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache', "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index f825819cb3e..16f3535d9f3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -297,6 +297,10 @@ class BuildPlugin implements Plugin { url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" } } + repos.maven { + name 'netty-snapshots' + url "http://s3.amazonaws.com/download.elasticsearch.org/nettysnapshots/20160722" + } } /** Returns a closure which can be used with a MavenPom for removing transitive dependencies. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy index 9bb8c4ae83d..fccbb43e872 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy @@ -23,6 +23,7 @@ import org.apache.rat.anttasks.SubstringLicenseMatcher import org.apache.rat.license.SimpleLicenseFamily import org.elasticsearch.gradle.AntTask import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.Input import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.SourceSet @@ -44,6 +45,16 @@ public class LicenseHeadersTask extends AntTask { */ protected List javaFiles + /** Allowed license families for this project. */ + @Input + List approvedLicenses = ['Apache', 'Generated'] + + /** + * Additional license families that may be found. The key is the license category name (5 characters), + * followed by the family name and the value list of patterns to search for. + */ + protected Map additionalLicenses = new HashMap<>() + LicenseHeadersTask() { description = "Checks sources for missing, incorrect, or unacceptable license headers" // Delay resolving the dependencies until after evaluation so we pick up generated sources @@ -53,6 +64,22 @@ public class LicenseHeadersTask extends AntTask { } } + /** + * Add a new license type. + * + * The license may be added to the {@link #approvedLicenses} using the {@code familyName}. + * + * @param categoryName A 5-character string identifier for the license + * @param familyName An expanded string name for the license + * @param pattern A pattern to search for, which if found, indicates a file contains the license + */ + public void additionalLicense(String categoryName, String familyName, String pattern) { + if (categoryName.length() != 5) { + throw new IllegalArgumentException("License category name must be exactly 5 characters, got ${categoryName}"); + } + additionalLicenses.put(categoryName + familyName, pattern); + } + @Override protected void runAnt(AntBuilder ant) { ant.project.addTaskDefinition('ratReport', Report) @@ -64,43 +91,54 @@ public class LicenseHeadersTask extends AntTask { // run rat, going to the file List input = javaFiles ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) { - for (FileCollection dirSet : input) { - for (File dir: dirSet.srcDirs) { - // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main... - if (dir.exists()) { - ant.fileset(dir: dir) - } + for (FileCollection dirSet : input) { + for (File dir: dirSet.srcDirs) { + // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main... + if (dir.exists()) { + ant.fileset(dir: dir) } } + } - // BSD 4-clause stuff (is disallowed below) - // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. - substringMatcher(licenseFamilyCategory: "BSD4 ", - licenseFamilyName: "Original BSD License (with advertising clause)") { - pattern(substring: "All advertising materials") - } + // BSD 4-clause stuff (is disallowed below) + // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. + substringMatcher(licenseFamilyCategory: "BSD4 ", + licenseFamilyName: "Original BSD License (with advertising clause)") { + pattern(substring: "All advertising materials") + } - // Apache - substringMatcher(licenseFamilyCategory: "AL ", - licenseFamilyName: "Apache") { - // Apache license (ES) - pattern(substring: "Licensed to Elasticsearch under one or more contributor") - // Apache license (ASF) - pattern(substring: "Licensed to the Apache Software Foundation (ASF) under") - // this is the old-school one under some files - pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")") - } + // Apache + substringMatcher(licenseFamilyCategory: "AL ", + licenseFamilyName: "Apache") { + // Apache license (ES) + pattern(substring: "Licensed to Elasticsearch under one or more contributor") + // Apache license (ASF) + pattern(substring: "Licensed to the Apache Software Foundation (ASF) under") + // this is the old-school one under some files + pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")") + } - // Generated resources - substringMatcher(licenseFamilyCategory: "GEN ", - licenseFamilyName: "Generated") { - // parsers generated by antlr - pattern(substring: "ANTLR GENERATED CODE") - } + // Generated resources + substringMatcher(licenseFamilyCategory: "GEN ", + licenseFamilyName: "Generated") { + // parsers generated by antlr + pattern(substring: "ANTLR GENERATED CODE") + } - // approved categories - approvedLicense(familyName: "Apache") - approvedLicense(familyName: "Generated") + // license types added by the project + for (Map.Entry additional : additionalLicenses.entrySet()) { + String category = additional.getKey().substring(0, 5) + String family = additional.getKey().substring(5) + substringMatcher(licenseFamilyCategory: category, + licenseFamilyName: family) { + pattern(substring: additional.getValue()) + } + } + + // approved categories + for (String licenseFamily : approvedLicenses) { + approvedLicense(familyName: licenseFamily) + } } // check the license file for any errors, this should be fast. diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index de5fe339a30..339abc8c02d 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -535,7 +535,6 @@ - @@ -1106,28 +1105,15 @@ - - - - - - - - - - - - - diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 7565488d4ab..9fb8d89e824 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -13,7 +13,7 @@ jna = 4.2.2 randomizedrunner = 2.3.2 junit = 4.11 httpclient = 4.5.2 -httpcore = 4.4.4 +httpcore = 4.4.5 commonslogging = 1.1.3 commonscodec = 1.10 hamcrest = 1.3 diff --git a/client/benchmark/README.md b/client/benchmark/README.md new file mode 100644 index 00000000000..9ff2a327b33 --- /dev/null +++ b/client/benchmark/README.md @@ -0,0 +1,35 @@ +Steps to execute the benchmark: + +1. Start Elasticsearch on the target host (ideally *not* on the same machine) +2. Create an empty index with the mapping you want to benchmark +3. Start either the RestClientBenchmark class or the TransportClientBenchmark +4. Delete the index +5. Repeat steps 2. - 4. for multiple iterations. The first iterations are intended as warmup for Elasticsearch itself. Always start the same benchmark in step 3! +4. After the benchmark: Shutdown Elasticsearch and delete the data directory + +Repeat all steps above for the other benchmark candidate. + +Example benchmark: + +* Download benchmark data from http://benchmarks.elastic.co/corpora/geonames/documents.json.bz2 and decompress +* Use the mapping file https://github.com/elastic/rally-tracks/blob/master/geonames/mappings.json to create the index + +Example command line parameter list: + +``` +192.168.2.2 /home/your_user_name/.rally/benchmarks/data/geonames/documents.json geonames type 8647880 5000 "{ \"query\": { \"match_phrase\": { \"name\": \"Sankt Georgen\" } } }\"" +``` + +The parameters are in order: + +* Benchmark target host IP (the host where Elasticsearch is running) +* full path to the file that should be bulk indexed +* name of the index +* name of the (sole) type in the index +* number of documents in the file +* bulk size +* a search request body (remember to escape double quotes). The `TransportClientBenchmark` uses `QueryBuilders.wrapperQuery()` internally which automatically adds a root key `query`, so it must not be present in the command line parameter. + +You should also define a few GC-related settings `-Xms4096M -Xmx4096M -XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails` and keep an eye on GC activity. You can also define `-XX:+PrintCompilation` to see JIT activity. + + diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle new file mode 100644 index 00000000000..ebaff611627 --- /dev/null +++ b/client/benchmark/build.gradle @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.build' + +group = 'org.elasticsearch.client' + +// never try to invoke tests on the benchmark project - there aren't any +check.dependsOn.remove(test) +// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip +task test(type: Test, overwrite: true) + +dependencies { + compile 'org.apache.commons:commons-math3:3.2' + + compile("org.elasticsearch.client:rest:${version}") + // for transport client + compile("org.elasticsearch:elasticsearch:${version}") + compile("org.elasticsearch.client:transport:${version}") + compile project(path: ':modules:transport-netty3', configuration: 'runtime') + compile project(path: ':modules:transport-netty4', configuration: 'runtime') + compile project(path: ':modules:reindex', configuration: 'runtime') + compile project(path: ':modules:lang-mustache', configuration: 'runtime') + compile project(path: ':modules:percolator', configuration: 'runtime') +} + +// No licenses for our benchmark deps (we don't ship benchmarks) +dependencyLicenses.enabled = false + +extraArchive { + javadoc = false +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java new file mode 100644 index 00000000000..dc018f03dc4 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark; + +import org.elasticsearch.client.benchmark.ops.bulk.BulkBenchmarkTask; +import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor; +import org.elasticsearch.client.benchmark.ops.search.SearchBenchmarkTask; +import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor; +import org.elasticsearch.common.SuppressForbidden; + +import java.io.Closeable; + +public abstract class AbstractBenchmark { + private static final int SEARCH_BENCHMARK_ITERATIONS = 10_000; + + protected abstract T client(String benchmarkTargetHost) throws Exception; + + protected abstract BulkRequestExecutor bulkRequestExecutor(T client, String indexName, String typeName); + + protected abstract SearchRequestExecutor searchRequestExecutor(T client, String indexName); + + @SuppressForbidden(reason = "system out is ok for a command line tool") + public final void run(String[] args) throws Exception { + if (args.length < 6) { + System.err.println( + "usage: benchmarkTargetHostIp indexFilePath indexName typeName numberOfDocuments bulkSize [search request body]"); + System.exit(1); + } + String benchmarkTargetHost = args[0]; + String indexFilePath = args[1]; + String indexName = args[2]; + String typeName = args[3]; + int totalDocs = Integer.valueOf(args[4]); + int bulkSize = Integer.valueOf(args[5]); + + int totalIterationCount = (int) Math.floor(totalDocs / bulkSize); + // consider 40% of all iterations as warmup iterations + int warmupIterations = (int) (0.4d * totalIterationCount); + int iterations = totalIterationCount - warmupIterations; + String searchBody = (args.length == 7) ? args[6] : null; + + T client = client(benchmarkTargetHost); + + BenchmarkRunner benchmark = new BenchmarkRunner(warmupIterations, iterations, + new BulkBenchmarkTask( + bulkRequestExecutor(client, indexName, typeName), indexFilePath, warmupIterations + iterations, bulkSize)); + + try { + benchmark.run(); + if (searchBody != null) { + for (int run = 1; run <= 5; run++) { + System.out.println("============="); + System.out.println(" Trial run " + run); + System.out.println("============="); + + for (int throughput = 100; throughput <= 100_000; throughput *= 10) { + //request a GC between trials to reduce the likelihood of a GC occurring in the middle of a trial. + System.gc(); + + BenchmarkRunner searchBenchmark = new BenchmarkRunner(SEARCH_BENCHMARK_ITERATIONS, SEARCH_BENCHMARK_ITERATIONS, + new SearchBenchmarkTask( + searchRequestExecutor(client, indexName), searchBody, 2 * SEARCH_BENCHMARK_ITERATIONS, throughput)); + System.out.printf("Target throughput = %d ops / s%n", throughput); + searchBenchmark.run(); + } + } + } + } finally { + client.close(); + } + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java new file mode 100644 index 00000000000..655b5815f35 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark; + +import org.elasticsearch.client.benchmark.metrics.Metrics; +import org.elasticsearch.client.benchmark.metrics.MetricsCalculator; +import org.elasticsearch.client.benchmark.metrics.Sample; +import org.elasticsearch.client.benchmark.metrics.SampleRecorder; +import org.elasticsearch.common.SuppressForbidden; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +public final class BenchmarkRunner { + private final int warmupIterations; + private final int iterations; + private final BenchmarkTask task; + + public BenchmarkRunner(int warmupIterations, int iterations, BenchmarkTask task) { + this.warmupIterations = warmupIterations; + this.iterations = iterations; + this.task = task; + } + + @SuppressForbidden(reason = "system out is ok for a command line tool") + public void run() throws Exception { + SampleRecorder recorder = new SampleRecorder(warmupIterations, iterations); + System.out.printf("Running %s with %d warmup iterations and %d iterations.%n", + task.getClass().getSimpleName(), warmupIterations, iterations); + + try { + task.setUp(recorder); + task.run(); + task.tearDown(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return; + } + + List samples = recorder.getSamples(); + final List summaryMetrics = MetricsCalculator.calculate(samples); + + if (summaryMetrics.isEmpty()) { + System.out.println("No results."); + } + + for (Metrics metrics : summaryMetrics) { + System.out.printf(Locale.ROOT, "Operation: %s%n", metrics.operation); + String stats = String.format(Locale.ROOT, + "Throughput = %f ops/s, p90 = %f ms, p95 = %f ms, p99 = %f ms, p99.9 = %f ms, p99.99 = %f ms", + metrics.throughput, + metrics.serviceTimeP90, metrics.serviceTimeP95, + metrics.serviceTimeP99, metrics.serviceTimeP999, + metrics.serviceTimeP9999); + System.out.println(repeat(stats.length(), '-')); + System.out.println(stats); + System.out.printf("success count = %d, error count = %d%n", metrics.successCount, metrics.errorCount); + System.out.println(repeat(stats.length(), '-')); + } + } + + private String repeat(int times, char character) { + char[] characters = new char[times]; + Arrays.fill(characters, character); + return new String(characters); + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java new file mode 100644 index 00000000000..5aabd0e1940 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark; + +import org.elasticsearch.client.benchmark.metrics.SampleRecorder; + +public interface BenchmarkTask { + void setUp(SampleRecorder sampleRecorder) throws Exception; + + void run() throws Exception; + + void tearDown() throws Exception; +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java new file mode 100644 index 00000000000..9108afe4446 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.metrics; + +public final class Metrics { + public final String operation; + public final long successCount; + public final long errorCount; + public final double throughput; + public final double serviceTimeP90; + public final double serviceTimeP95; + public final double serviceTimeP99; + public final double serviceTimeP999; + public final double serviceTimeP9999; + + public Metrics(String operation, long successCount, long errorCount, double throughput, + double serviceTimeP90, double serviceTimeP95, double serviceTimeP99, + double serviceTimeP999, double serviceTimeP9999) { + this.operation = operation; + this.successCount = successCount; + this.errorCount = errorCount; + this.throughput = throughput; + this.serviceTimeP90 = serviceTimeP90; + this.serviceTimeP95 = serviceTimeP95; + this.serviceTimeP99 = serviceTimeP99; + this.serviceTimeP999 = serviceTimeP999; + this.serviceTimeP9999 = serviceTimeP9999; + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java new file mode 100644 index 00000000000..5b455127f52 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.metrics; + +import org.apache.commons.math3.stat.StatUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public final class MetricsCalculator { + public static List calculate(Collection samples) { + Map> samplesPerOperation = groupByOperation(samples); + return calculateMetricsPerOperation(samplesPerOperation); + } + + private static Map> groupByOperation(Collection samples) { + Map> samplesPerOperation = new HashMap<>(); + + for (Sample sample : samples) { + if (!samplesPerOperation.containsKey(sample.getOperation())) { + samplesPerOperation.put(sample.getOperation(), new ArrayList<>()); + } + samplesPerOperation.get(sample.getOperation()).add(sample); + } + return samplesPerOperation; + } + + private static List calculateMetricsPerOperation(Map> samplesPerOperation) { + List metrics = new ArrayList<>(); + for (Map.Entry> operationAndMetrics : samplesPerOperation.entrySet()) { + List samples = operationAndMetrics.getValue(); + double[] serviceTimes = new double[samples.size()]; + int it = 0; + long firstStart = Long.MAX_VALUE; + long latestEnd = Long.MIN_VALUE; + for (Sample sample : samples) { + firstStart = Math.min(sample.getStartTimestamp(), firstStart); + latestEnd = Math.max(sample.getStopTimestamp(), latestEnd); + serviceTimes[it++] = sample.getServiceTime(); + } + + metrics.add(new Metrics(operationAndMetrics.getKey(), + samples.stream().filter((r) -> r.isSuccess()).count(), + samples.stream().filter((r) -> !r.isSuccess()).count(), + // throughput calculation is based on the total (Wall clock) time it took to generate all samples + calculateThroughput(samples.size(), latestEnd - firstStart), + // convert ns -> ms without losing precision + StatUtils.percentile(serviceTimes, 90.0d) / TimeUnit.MILLISECONDS.toNanos(1L), + StatUtils.percentile(serviceTimes, 95.0d) / TimeUnit.MILLISECONDS.toNanos(1L), + StatUtils.percentile(serviceTimes, 99.0d) / TimeUnit.MILLISECONDS.toNanos(1L), + StatUtils.percentile(serviceTimes, 99.9d) / TimeUnit.MILLISECONDS.toNanos(1L), + StatUtils.percentile(serviceTimes, 99.99d) / TimeUnit.MILLISECONDS.toNanos(1L))); + } + return metrics; + } + + private static double calculateThroughput(int sampleSize, double duration) { + return sampleSize * (TimeUnit.SECONDS.toNanos(1L) / duration); + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java new file mode 100644 index 00000000000..59cd6bfd101 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.metrics; + +public final class Sample { + private final String operation; + private final long startTimestamp; + private final long stopTimestamp; + private final boolean success; + + public Sample(String operation, long startTimestamp, long stopTimestamp, boolean success) { + this.operation = operation; + this.startTimestamp = startTimestamp; + this.stopTimestamp = stopTimestamp; + this.success = success; + } + + public String getOperation() { + return operation; + } + + public boolean isSuccess() { + return success; + } + + public long getStartTimestamp() { + return startTimestamp; + } + + public long getStopTimestamp() { + return stopTimestamp; + } + + public long getServiceTime() { + // this is *not* latency, we're not including wait time in the queue (on purpose) + return stopTimestamp - startTimestamp; + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java new file mode 100644 index 00000000000..d9f24aea004 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.metrics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Stores measurement samples. + * + * This class is NOT threadsafe. + */ +public final class SampleRecorder { + private final int warmupIterations; + private final List samples; + private int currentIteration; + + public SampleRecorder(int warmupIterations, int iterations) { + this.warmupIterations = warmupIterations; + this.samples = new ArrayList<>(iterations); + } + + public void addSample(Sample sample) { + currentIteration++; + // only add samples after warmup + if (currentIteration > warmupIterations) { + samples.add(sample); + } + } + + public List getSamples() { + return Collections.unmodifiableList(samples); + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java new file mode 100644 index 00000000000..5844103fd1e --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.ops.bulk; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.benchmark.BenchmarkTask; +import org.elasticsearch.client.benchmark.metrics.Sample; +import org.elasticsearch.client.benchmark.metrics.SampleRecorder; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class BulkBenchmarkTask implements BenchmarkTask { + private final BulkRequestExecutor requestExecutor; + private final String indexFilePath; + private final int totalIterations; + private final int bulkSize; + private LoadGenerator generator; + private ExecutorService executorService; + + public BulkBenchmarkTask(BulkRequestExecutor requestExecutor, String indexFilePath, int totalIterations, int bulkSize) { + this.requestExecutor = requestExecutor; + this.indexFilePath = indexFilePath; + this.totalIterations = totalIterations; + this.bulkSize = bulkSize; + } + + @Override + @SuppressForbidden(reason = "PathUtils#get is fine - we don't have environment here") + public void setUp(SampleRecorder sampleRecorder) { + BlockingQueue> bulkQueue = new ArrayBlockingQueue<>(256); + + BulkIndexer runner = new BulkIndexer(bulkQueue, totalIterations, sampleRecorder, requestExecutor); + + executorService = Executors.newSingleThreadExecutor((r) -> new Thread(r, "bulk-index-runner")); + executorService.submit(runner); + + generator = new LoadGenerator(PathUtils.get(indexFilePath), bulkQueue, bulkSize); + } + + @Override + @SuppressForbidden(reason = "system out is ok for a command line tool") + public void run() throws Exception { + generator.execute(); + // when the generator is done, there are no more data -> shutdown client + executorService.shutdown(); + //We need to wait until the queue is drained + final boolean finishedNormally = executorService.awaitTermination(20, TimeUnit.MINUTES); + if (finishedNormally == false) { + System.err.println("Background tasks are still running after timeout on enclosing pool. Forcing pool shutdown."); + executorService.shutdownNow(); + } + } + + @Override + public void tearDown() { + //no op + } + + private static final class LoadGenerator { + private final Path bulkDataFile; + private final BlockingQueue> bulkQueue; + private final int bulkSize; + + public LoadGenerator(Path bulkDataFile, BlockingQueue> bulkQueue, int bulkSize) { + this.bulkDataFile = bulkDataFile; + this.bulkQueue = bulkQueue; + this.bulkSize = bulkSize; + } + + @SuppressForbidden(reason = "Classic I/O is fine in non-production code") + public void execute() { + try (BufferedReader reader = Files.newBufferedReader(bulkDataFile, StandardCharsets.UTF_8)) { + String line; + int bulkIndex = 0; + List bulkData = new ArrayList<>(bulkSize); + while ((line = reader.readLine()) != null) { + if (bulkIndex == bulkSize) { + sendBulk(bulkData); + // reset data structures + bulkData = new ArrayList<>(bulkSize); + bulkIndex = 0; + } + bulkData.add(line); + bulkIndex++; + } + // also send the last bulk: + if (bulkIndex > 0) { + sendBulk(bulkData); + } + } catch (IOException e) { + throw new ElasticsearchException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void sendBulk(List bulkData) throws InterruptedException { + bulkQueue.put(bulkData); + } + } + + + private static final class BulkIndexer implements Runnable { + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkIndexer.class.getName()); + + private final BlockingQueue> bulkData; + private final int totalIterations; + private final BulkRequestExecutor bulkRequestExecutor; + private final SampleRecorder sampleRecorder; + + public BulkIndexer(BlockingQueue> bulkData, int totalIterations, SampleRecorder sampleRecorder, + BulkRequestExecutor bulkRequestExecutor) { + this.bulkData = bulkData; + this.totalIterations = totalIterations; + this.bulkRequestExecutor = bulkRequestExecutor; + this.sampleRecorder = sampleRecorder; + } + + @Override + public void run() { + for (int iteration = 0; iteration < totalIterations; iteration++) { + boolean success = false; + List currentBulk; + try { + currentBulk = bulkData.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + // Yes, this approach is prone to coordinated omission *but* we have to consider that we want to benchmark a closed system + // with backpressure here instead of an open system. So this is actually correct in this case. + long start = System.nanoTime(); + try { + success = bulkRequestExecutor.bulkIndex(currentBulk); + } catch (Exception ex) { + logger.warn("Error while executing bulk request", ex); + } + long stop = System.nanoTime(); + sampleRecorder.addSample(new Sample("bulk", start, stop, success)); + } + } + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java new file mode 100644 index 00000000000..be95153132f --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.ops.bulk; + +import java.util.List; + +public interface BulkRequestExecutor { + boolean bulkIndex(List bulkData); +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java new file mode 100644 index 00000000000..1476e9ebba5 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.ops.search; + +import org.elasticsearch.client.benchmark.BenchmarkTask; +import org.elasticsearch.client.benchmark.metrics.Sample; +import org.elasticsearch.client.benchmark.metrics.SampleRecorder; + +import java.util.concurrent.TimeUnit; + +public class SearchBenchmarkTask implements BenchmarkTask { + private static final long MICROS_PER_SEC = TimeUnit.SECONDS.toMicros(1L); + private static final long NANOS_PER_MICRO = TimeUnit.MICROSECONDS.toNanos(1L); + + private final SearchRequestExecutor searchRequestExecutor; + private final String searchRequestBody; + private final int iterations; + private final int targetThroughput; + + private SampleRecorder sampleRecorder; + + public SearchBenchmarkTask(SearchRequestExecutor searchRequestExecutor, String body, int iterations, int targetThroughput) { + this.searchRequestExecutor = searchRequestExecutor; + this.searchRequestBody = body; + this.iterations = iterations; + this.targetThroughput = targetThroughput; + } + + @Override + public void setUp(SampleRecorder sampleRecorder) throws Exception { + this.sampleRecorder = sampleRecorder; + } + + @Override + public void run() throws Exception { + for (int iteration = 0; iteration < this.iterations; iteration++) { + final long start = System.nanoTime(); + boolean success = searchRequestExecutor.search(searchRequestBody); + final long stop = System.nanoTime(); + sampleRecorder.addSample(new Sample("search", start, stop, success)); + + int waitTime = (int) Math.floor(MICROS_PER_SEC / targetThroughput - (stop - start) / NANOS_PER_MICRO); + if (waitTime > 0) { + // Thread.sleep() time is not very accurate (it's most of the time around 1 - 2 ms off) + // so we rather busy spin for the last few microseconds. Still not entirely accurate but way closer + waitMicros(waitTime); + } + } + } + + private void waitMicros(int waitTime) throws InterruptedException { + int millis = waitTime / 1000; + int micros = waitTime % 1000; + if (millis > 0) { + Thread.sleep(millis); + } + // busy spin for the rest of the time + if (micros > 0) { + long end = System.nanoTime() + 1000L * micros; + while (end > System.nanoTime()) { + // busy spin + } + } + } + + @Override + public void tearDown() throws Exception { + // no op + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java new file mode 100644 index 00000000000..abdfecf73c6 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.ops.search; + +public interface SearchRequestExecutor { + boolean search(String source); +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java new file mode 100644 index 00000000000..bf661fa661f --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.rest; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.benchmark.AbstractBenchmark; +import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor; +import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public final class RestClientBenchmark extends AbstractBenchmark { + public static void main(String[] args) throws Exception { + RestClientBenchmark b = new RestClientBenchmark(); + b.run(args); + } + + @Override + protected RestClient client(String benchmarkTargetHost) { + return RestClient.builder(new HttpHost(benchmarkTargetHost, 9200)).build(); + } + + @Override + protected BulkRequestExecutor bulkRequestExecutor(RestClient client, String indexName, String typeName) { + return new RestBulkRequestExecutor(client, indexName, typeName); + } + + @Override + protected SearchRequestExecutor searchRequestExecutor(RestClient client, String indexName) { + return new RestSearchRequestExecutor(client, indexName); + } + + private static final class RestBulkRequestExecutor implements BulkRequestExecutor { + private final RestClient client; + private final String actionMetaData; + + public RestBulkRequestExecutor(RestClient client, String index, String type) { + this.client = client; + this.actionMetaData = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type); + } + + @Override + public boolean bulkIndex(List bulkData) { + StringBuilder bulkRequestBody = new StringBuilder(); + for (String bulkItem : bulkData) { + bulkRequestBody.append(actionMetaData); + bulkRequestBody.append(bulkItem); + bulkRequestBody.append("\n"); + } + HttpEntity entity = new NStringEntity(bulkRequestBody.toString(), ContentType.APPLICATION_JSON); + try { + Response response = client.performRequest("POST", "/geonames/type/_bulk", Collections.emptyMap(), entity); + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; + } catch (Exception e) { + throw new ElasticsearchException(e); + } + } + } + + private static final class RestSearchRequestExecutor implements SearchRequestExecutor { + private final RestClient client; + private final String endpoint; + + private RestSearchRequestExecutor(RestClient client, String indexName) { + this.client = client; + this.endpoint = "/" + indexName + "/_search"; + } + + @Override + public boolean search(String source) { + HttpEntity searchBody = new NStringEntity(source, StandardCharsets.UTF_8); + try { + Response response = client.performRequest("GET", endpoint, Collections.emptyMap(), searchBody); + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; + } catch (IOException e) { + throw new ElasticsearchException(e); + } + } + } +} diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java new file mode 100644 index 00000000000..c52414cf3a4 --- /dev/null +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.benchmark.transport; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.benchmark.AbstractBenchmark; +import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor; +import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.client.PreBuiltTransportClient; + +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public final class TransportClientBenchmark extends AbstractBenchmark { + public static void main(String[] args) throws Exception { + TransportClientBenchmark benchmark = new TransportClientBenchmark(); + benchmark.run(args); + } + + @Override + protected TransportClient client(String benchmarkTargetHost) throws Exception { + TransportClient client = new PreBuiltTransportClient(Settings.EMPTY); + client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(benchmarkTargetHost), 9300)); + return client; + } + + @Override + protected BulkRequestExecutor bulkRequestExecutor(TransportClient client, String indexName, String typeName) { + return new TransportBulkRequestExecutor(client, indexName, typeName); + } + + @Override + protected SearchRequestExecutor searchRequestExecutor(TransportClient client, String indexName) { + return new TransportSearchRequestExecutor(client, indexName); + } + + private static final class TransportBulkRequestExecutor implements BulkRequestExecutor { + private final TransportClient client; + private final String indexName; + private final String typeName; + + public TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) { + this.client = client; + this.indexName = indexName; + this.typeName = typeName; + } + + @Override + public boolean bulkIndex(List bulkData) { + BulkRequestBuilder builder = client.prepareBulk(); + for (String bulkItem : bulkData) { + builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8))); + } + BulkResponse bulkResponse; + try { + bulkResponse = builder.execute().get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException e) { + throw new ElasticsearchException(e); + } + return !bulkResponse.hasFailures(); + } + } + + private static final class TransportSearchRequestExecutor implements SearchRequestExecutor { + private final TransportClient client; + private final String indexName; + + private TransportSearchRequestExecutor(TransportClient client, String indexName) { + this.client = client; + this.indexName = indexName; + } + + @Override + public boolean search(String source) { + final SearchResponse response; + try { + response = client.prepareSearch(indexName).setQuery(QueryBuilders.wrapperQuery(source)).execute().get(); + return response.status() == RestStatus.OK; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException e) { + throw new ElasticsearchException(e); + } + } + } +} diff --git a/client/benchmark/src/main/resources/log4j.properties b/client/benchmark/src/main/resources/log4j.properties new file mode 100644 index 00000000000..22f54ef68e5 --- /dev/null +++ b/client/benchmark/src/main/resources/log4j.properties @@ -0,0 +1,9 @@ +es.logger.level=INFO +log4j.rootLogger=${es.logger.level}, out + +log4j.logger.org.apache.http=INFO, out +log4j.additivity.org.apache.http=false + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 4623fbd8c2c..fabe5c65373 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -31,6 +31,8 @@ group = 'org.elasticsearch.client' dependencies { compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "org.apache.httpcomponents:httpasyncclient:4.1.2" + compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}" compile "commons-codec:commons-codec:${versions.commonscodec}" compile "commons-logging:commons-logging:${versions.commonslogging}" @@ -56,6 +58,11 @@ forbiddenApisTest { signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] } +dependencyLicenses { + mapping from: /http.*/, to: 'httpclient' + mapping from: /commons-.*/, to: 'commons' +} + //JarHell is part of es core, which we don't want to pull in jarHell.enabled=false diff --git a/client/rest/licenses/commons-codec-LICENSE.txt b/client/rest/licenses/commons-LICENSE.txt similarity index 100% rename from client/rest/licenses/commons-codec-LICENSE.txt rename to client/rest/licenses/commons-LICENSE.txt diff --git a/client/rest/licenses/commons-codec-NOTICE.txt b/client/rest/licenses/commons-NOTICE.txt similarity index 100% rename from client/rest/licenses/commons-codec-NOTICE.txt rename to client/rest/licenses/commons-NOTICE.txt diff --git a/client/rest/licenses/commons-logging-NOTICE.txt b/client/rest/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 556bd03951d..00000000000 --- a/client/rest/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - diff --git a/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 b/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 new file mode 100644 index 00000000000..065ed920a17 --- /dev/null +++ b/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 @@ -0,0 +1 @@ +95aa3e6fb520191a0970a73cf09f62948ee614be \ No newline at end of file diff --git a/client/rest/licenses/httpcore-4.4.4.jar.sha1 b/client/rest/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/client/rest/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-4.4.5.jar.sha1 b/client/rest/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/client/rest/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-LICENSE.txt b/client/rest/licenses/httpcore-LICENSE.txt deleted file mode 100644 index 32f01eda18f..00000000000 --- a/client/rest/licenses/httpcore-LICENSE.txt +++ /dev/null @@ -1,558 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -========================================================================= - -This project includes Public Suffix List copied from - -licensed under the terms of the Mozilla Public License, v. 2.0 - -Full license text: - -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. diff --git a/client/rest/licenses/httpcore-NOTICE.txt b/client/rest/licenses/httpcore-NOTICE.txt deleted file mode 100644 index 91e5c40c4c6..00000000000 --- a/client/rest/licenses/httpcore-NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache HttpComponents Client -Copyright 1999-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - diff --git a/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 b/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 new file mode 100644 index 00000000000..d6a80bf100d --- /dev/null +++ b/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 @@ -0,0 +1 @@ +f4be009e7505f6ceddf21e7960c759f413f15056 \ No newline at end of file diff --git a/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java b/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java new file mode 100644 index 00000000000..da7f5c79721 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumer.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.ContentTooLongException; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpResponse; +import org.apache.http.entity.ContentType; +import org.apache.http.nio.ContentDecoder; +import org.apache.http.nio.IOControl; +import org.apache.http.nio.entity.ContentBufferEntity; +import org.apache.http.nio.protocol.AbstractAsyncResponseConsumer; +import org.apache.http.nio.util.ByteBufferAllocator; +import org.apache.http.nio.util.HeapByteBufferAllocator; +import org.apache.http.nio.util.SimpleInputBuffer; +import org.apache.http.protocol.HttpContext; + +import java.io.IOException; + +/** + * Default implementation of {@link org.apache.http.nio.protocol.HttpAsyncResponseConsumer}. Buffers the whole + * response content in heap memory, meaning that the size of the buffer is equal to the content-length of the response. + * Limits the size of responses that can be read to {@link #DEFAULT_BUFFER_LIMIT} by default, configurable value. + * Throws an exception in case the entity is longer than the configured buffer limit. + */ +public class HeapBufferedAsyncResponseConsumer extends AbstractAsyncResponseConsumer { + + //default buffer limit is 10MB + public static final int DEFAULT_BUFFER_LIMIT = 10 * 1024 * 1024; + + private final int bufferLimit; + private volatile HttpResponse response; + private volatile SimpleInputBuffer buf; + + /** + * Creates a new instance of this consumer with a buffer limit of {@link #DEFAULT_BUFFER_LIMIT} + */ + public HeapBufferedAsyncResponseConsumer() { + this.bufferLimit = DEFAULT_BUFFER_LIMIT; + } + + /** + * Creates a new instance of this consumer with the provided buffer limit + */ + public HeapBufferedAsyncResponseConsumer(int bufferLimit) { + if (bufferLimit <= 0) { + throw new IllegalArgumentException("bufferLimit must be greater than 0"); + } + this.bufferLimit = bufferLimit; + } + + @Override + protected void onResponseReceived(HttpResponse response) throws HttpException, IOException { + this.response = response; + } + + @Override + protected void onEntityEnclosed(HttpEntity entity, ContentType contentType) throws IOException { + long len = entity.getContentLength(); + if (len > bufferLimit) { + throw new ContentTooLongException("entity content is too long [" + len + + "] for the configured buffer limit [" + bufferLimit + "]"); + } + if (len < 0) { + len = 4096; + } + this.buf = new SimpleInputBuffer((int) len, getByteBufferAllocator()); + this.response.setEntity(new ContentBufferEntity(entity, this.buf)); + } + + /** + * Returns the instance of {@link ByteBufferAllocator} to use for content buffering. + * Allows to plug in any {@link ByteBufferAllocator} implementation. + */ + protected ByteBufferAllocator getByteBufferAllocator() { + return HeapByteBufferAllocator.INSTANCE; + } + + @Override + protected void onContentReceived(ContentDecoder decoder, IOControl ioctrl) throws IOException { + this.buf.consumeContent(decoder); + } + + @Override + protected HttpResponse buildResult(HttpContext context) throws Exception { + return response; + } + + @Override + protected void releaseResources() { + response = null; + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 24e6881fa1e..fd4c3600234 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -81,7 +81,7 @@ final class RequestLogger { /** * Logs a request that failed */ - static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, IOException e) { + static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, Exception e) { if (logger.isDebugEnabled()) { logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + "] failed", e); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/Response.java b/client/rest/src/main/java/org/elasticsearch/client/Response.java index f7685b27bb9..7edaa49c2ea 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Response.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Response.java @@ -22,26 +22,23 @@ package org.elasticsearch.client; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; import org.apache.http.RequestLine; import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import java.io.Closeable; -import java.io.IOException; import java.util.Objects; /** - * Holds an elasticsearch response. It wraps the {@link CloseableHttpResponse} response and associates it with + * Holds an elasticsearch response. It wraps the {@link HttpResponse} returned and associates it with * its corresponding {@link RequestLine} and {@link HttpHost}. - * It must be closed to free any resource held by it, as well as the corresponding connection in the connection pool. */ -public class Response implements Closeable { +public final class Response { private final RequestLine requestLine; private final HttpHost host; - private final CloseableHttpResponse response; + private final HttpResponse response; - Response(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) { + Response(RequestLine requestLine, HttpHost host, HttpResponse response) { Objects.requireNonNull(requestLine, "requestLine cannot be null"); Objects.requireNonNull(host, "node cannot be null"); Objects.requireNonNull(response, "response cannot be null"); @@ -107,9 +104,4 @@ public class Response implements Closeable { ", response=" + response.getStatusLine() + '}'; } - - @Override - public void close() throws IOException { - this.response.close(); - } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java index 44f59cce7db..35e31d921c7 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java +++ b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java @@ -23,44 +23,26 @@ import java.io.IOException; /** * Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error. - * Note that the response body gets passed in as a string and read eagerly, which means that the Response object - * is expected to be closed and available only to read metadata like status line, request line, response headers. + * Holds the response that was returned. */ -public class ResponseException extends IOException { +public final class ResponseException extends IOException { private Response response; - private final String responseBody; - ResponseException(Response response, String responseBody) throws IOException { - super(buildMessage(response,responseBody)); + ResponseException(Response response) throws IOException { + super(buildMessage(response)); this.response = response; - this.responseBody = responseBody; } - private static String buildMessage(Response response, String responseBody) { - String message = response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri() + private static String buildMessage(Response response) { + return response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri() + ": " + response.getStatusLine().toString(); - if (responseBody != null) { - message += "\n" + responseBody; - } - return message; } /** * Returns the {@link Response} that caused this exception to be thrown. - * Expected to be used only to read metadata like status line, request line, response headers. The response body should - * be retrieved using {@link #getResponseBody()} */ public Response getResponse() { return response; } - - /** - * Returns the response body as a string or null if there wasn't any. - * The body is eagerly consumed when an ResponseException gets created, and its corresponding Response - * gets closed straightaway so this method is the only way to get back the response body that was returned. - */ - public String getResponseBody() { - return responseBody; - } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java new file mode 100644 index 00000000000..ce948f6569b --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/ResponseListener.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +/** + * Listener to be provided when calling async performRequest methods provided by {@link RestClient}. + * Those methods that do accept a listener will return immediately, execute asynchronously, and notify + * the listener whenever the request yielded a response, or failed with an exception. + */ +public interface ResponseListener { + + /** + * Method invoked if the request yielded a successful response + */ + void onSuccess(Response response); + + /** + * Method invoked if the request failed. There are two main categories of failures: connection failures (usually + * {@link java.io.IOException}s, or responses that were treated as errors based on their error response code + * ({@link ResponseException}s). + */ + void onFailure(Exception exception); +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 0d6941c7857..84863580bb1 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -20,14 +20,12 @@ package org.elasticsearch.client; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.Consts; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpOptions; @@ -37,20 +35,22 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.methods.HttpTrace; import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.ContentType; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.util.EntityUtils; +import org.apache.http.concurrent.FutureCallback; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.nio.client.methods.HttpAsyncMethods; +import org.apache.http.nio.protocol.HttpAsyncRequestProducer; +import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import java.io.Closeable; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -58,30 +58,31 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * Client that connects to an elasticsearch cluster through http. - * Must be created using {@link Builder}, which allows to set all the different options or just rely on defaults. + * Must be created using {@link RestClientBuilder}, which allows to set all the different options or just rely on defaults. * The hosts that are part of the cluster need to be provided at creation time, but can also be replaced later * by calling {@link #setHosts(HttpHost...)}. * The method {@link #performRequest(String, String, Map, HttpEntity, Header...)} allows to send a request to the cluster. When * sending a request, a host gets selected out of the provided ones in a round-robin fashion. Failing hosts are marked dead and * retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously * failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead nodes that - * deserve a retry) are retried till one responds or none of them does, in which case an {@link IOException} will be thrown. + * deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown. * * Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format. */ public final class RestClient implements Closeable { private static final Log logger = LogFactory.getLog(RestClient.class); - public static ContentType JSON_CONTENT_TYPE = ContentType.create("application/json", Consts.UTF_8); - private final CloseableHttpClient client; - //we don't rely on default headers supported by HttpClient as those cannot be replaced, plus it would get hairy - //when we create the HttpClient instance on our own as there would be two different ways to set the default headers. + private final CloseableHttpAsyncClient client; + //we don't rely on default headers supported by HttpAsyncClient as those cannot be replaced private final Header[] defaultHeaders; private final long maxRetryTimeoutMillis; private final AtomicInteger lastHostIndex = new AtomicInteger(0); @@ -89,7 +90,7 @@ public final class RestClient implements Closeable { private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); private final FailureListener failureListener; - RestClient(CloseableHttpClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, + RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, HttpHost[] hosts, FailureListener failureListener) { this.client = client; this.maxRetryTimeoutMillis = maxRetryTimeoutMillis; @@ -98,6 +99,13 @@ public final class RestClient implements Closeable { setHosts(hosts); } + /** + * Returns a new {@link RestClientBuilder} to help with {@link RestClient} creation. + */ + public static RestClientBuilder builder(HttpHost... hosts) { + return new RestClientBuilder(hosts); + } + /** * Replaces the hosts that the client communicates with. * @see HttpHost @@ -116,8 +124,9 @@ public final class RestClient implements Closeable { } /** - * Sends a request to the elasticsearch cluster that the current client points to. - * Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters and request body. + * Sends a request to the elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters + * and request body. * * @param method the http method * @param endpoint the path of the request (without host and port) @@ -128,12 +137,12 @@ public final class RestClient implements Closeable { * @throws ResponseException in case elasticsearch responded with a status code that indicated an error */ public Response performRequest(String method, String endpoint, Header... headers) throws IOException { - return performRequest(method, endpoint, Collections.emptyMap(), null, headers); + return performRequest(method, endpoint, Collections.emptyMap(), (HttpEntity)null, headers); } /** - * Sends a request to the elasticsearch cluster that the current client points to. - * Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body. + * Sends a request to the elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body. * * @param method the http method * @param endpoint the path of the request (without host and port) @@ -145,15 +154,14 @@ public final class RestClient implements Closeable { * @throws ResponseException in case elasticsearch responded with a status code that indicated an error */ public Response performRequest(String method, String endpoint, Map params, Header... headers) throws IOException { - return performRequest(method, endpoint, params, null, headers); + return performRequest(method, endpoint, params, (HttpEntity)null, headers); } /** - * Sends a request to the elasticsearch cluster that the current client points to. - * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain - * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures, - * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried - * till one responds or none of them does, in which case an {@link IOException} will be thrown. + * Sends a request to the elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumer, Header...)} + * which doesn't require specifying an {@link HttpAsyncResponseConsumer} instance, {@link HeapBufferedAsyncResponseConsumer} + * will be used to consume the response body. * * @param method the http method * @param endpoint the path of the request (without host and port) @@ -167,72 +175,183 @@ public final class RestClient implements Closeable { */ public Response performRequest(String method, String endpoint, Map params, HttpEntity entity, Header... headers) throws IOException { + HttpAsyncResponseConsumer responseConsumer = new HeapBufferedAsyncResponseConsumer(); + return performRequest(method, endpoint, params, entity, responseConsumer, headers); + } + + /** + * Sends a request to the elasticsearch cluster that the client points to. Blocks until the request is completed and returns + * its response or fails by throwing an exception. Selects a host out of the provided ones in a round-robin fashion. Failing hosts + * are marked dead and retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times + * they previously failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead + * nodes that deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param responseConsumer the {@link HttpAsyncResponseConsumer} callback. Controls how the response + * body gets streamed from a non-blocking HTTP connection on the client side. + * @param headers the optional request headers + * @return the response returned by elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case elasticsearch responded with a status code that indicated an error + */ + public Response performRequest(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumer responseConsumer, + Header... headers) throws IOException { + SyncResponseListener listener = new SyncResponseListener(maxRetryTimeoutMillis); + performRequest(method, endpoint, params, entity, responseConsumer, listener, headers); + return listener.get(); + } + + /** + * Sends a request to the elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to + * {@link #performRequest(String, String, Map, HttpEntity, ResponseListener, Header...)} but without parameters and request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + public void performRequest(String method, String endpoint, ResponseListener responseListener, Header... headers) { + performRequest(method, endpoint, Collections.emptyMap(), null, responseListener, headers); + } + + /** + * Sends a request to the elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to + * {@link #performRequest(String, String, Map, HttpEntity, ResponseListener, Header...)} but without request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + public void performRequest(String method, String endpoint, Map params, + ResponseListener responseListener, Header... headers) { + performRequest(method, endpoint, params, null, responseListener, headers); + } + + /** + * Sends a request to the elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. + * Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumer, ResponseListener, Header...)} + * which doesn't require specifying an {@link HttpAsyncResponseConsumer} instance, {@link HeapBufferedAsyncResponseConsumer} + * will be used to consume the response body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + public void performRequest(String method, String endpoint, Map params, + HttpEntity entity, ResponseListener responseListener, Header... headers) { + HttpAsyncResponseConsumer responseConsumer = new HeapBufferedAsyncResponseConsumer(); + performRequest(method, endpoint, params, entity, responseConsumer, responseListener, headers); + } + + /** + * Sends a request to the elasticsearch cluster that the client points to. The request is executed asynchronously + * and the provided {@link ResponseListener} gets notified upon request completion or failure. + * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain + * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures, + * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried + * until one responds or none of them does, in which case an {@link IOException} will be thrown. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param responseConsumer the {@link HttpAsyncResponseConsumer} callback. Controls how the response + * body gets streamed from a non-blocking HTTP connection on the client side. + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + public void performRequest(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumer responseConsumer, + ResponseListener responseListener, Header... headers) { URI uri = buildUri(endpoint, params); HttpRequestBase request = createHttpRequest(method, uri, entity); setHeaders(request, headers); - //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt - long retryTimeoutMillis = Math.round(this.maxRetryTimeoutMillis / (float)100 * 98); - IOException lastSeenException = null; + FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener); long startTime = System.nanoTime(); - for (HttpHost host : nextHost()) { - if (lastSeenException != null) { - //in case we are retrying, check whether maxRetryTimeout has been reached, in which case an exception will be thrown - long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); - long timeout = retryTimeoutMillis - timeElapsedMillis; - if (timeout <= 0) { - IOException retryTimeoutException = new IOException( - "request retries exceeded max retry timeout [" + retryTimeoutMillis + "]"); - retryTimeoutException.addSuppressed(lastSeenException); - throw retryTimeoutException; + performRequest(startTime, nextHost().iterator(), request, responseConsumer, failureTrackingResponseListener); + } + + private void performRequest(final long startTime, final Iterator hosts, final HttpRequestBase request, + final HttpAsyncResponseConsumer responseConsumer, + final FailureTrackingResponseListener listener) { + final HttpHost host = hosts.next(); + //we stream the request body if the entity allows for it + HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request); + client.execute(requestProducer, responseConsumer, new FutureCallback() { + @Override + public void completed(HttpResponse httpResponse) { + try { + RequestLogger.logResponse(logger, request, host, httpResponse); + int statusCode = httpResponse.getStatusLine().getStatusCode(); + Response response = new Response(request.getRequestLine(), host, httpResponse); + if (isSuccessfulResponse(request.getMethod(), statusCode)) { + onResponse(host); + listener.onSuccess(response); + } else { + ResponseException responseException = new ResponseException(response); + if (isRetryStatus(statusCode)) { + //mark host dead and retry against next one + onFailure(host); + retryIfPossible(responseException, hosts, request); + } else { + //mark host alive and don't retry, as the error should be a request problem + onResponse(host); + listener.onDefinitiveFailure(responseException); + } + } + } catch(Exception e) { + listener.onDefinitiveFailure(e); } - //also reset the request to make it reusable for the next attempt - request.reset(); } - CloseableHttpResponse httpResponse; - try { - httpResponse = client.execute(host, request); - } catch(IOException e) { - RequestLogger.logFailedRequest(logger, request, host, e); - onFailure(host); - lastSeenException = addSuppressedException(lastSeenException, e); - continue; - } - Response response = new Response(request.getRequestLine(), host, httpResponse); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) { - RequestLogger.logResponse(logger, request, host, httpResponse); - onResponse(host); - return response; - } - RequestLogger.logResponse(logger, request, host, httpResponse); - String responseBody; - try { - if (response.getEntity() == null) { - responseBody = null; - } else { - responseBody = EntityUtils.toString(response.getEntity()); - } - } finally { - response.close(); - } - lastSeenException = addSuppressedException(lastSeenException, new ResponseException(response, responseBody)); - switch(statusCode) { - case 502: - case 503: - case 504: - //mark host dead and retry against next one + @Override + public void failed(Exception failure) { + try { + RequestLogger.logFailedRequest(logger, request, host, failure); onFailure(host); - break; - default: - //mark host alive and don't retry, as the error should be a request problem - onResponse(host); - throw lastSeenException; + retryIfPossible(failure, hosts, request); + } catch(Exception e) { + listener.onDefinitiveFailure(e); + } } - } - //we get here only when we tried all nodes and they all failed - assert lastSeenException != null; - throw lastSeenException; + + private void retryIfPossible(Exception exception, Iterator hosts, HttpRequestBase request) { + if (hosts.hasNext()) { + //in case we are retrying, check whether maxRetryTimeout has been reached + long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); + long timeout = maxRetryTimeoutMillis - timeElapsedMillis; + if (timeout <= 0) { + IOException retryTimeoutException = new IOException( + "request retries exceeded max retry timeout [" + maxRetryTimeoutMillis + "]"); + listener.onDefinitiveFailure(retryTimeoutException); + } else { + listener.trackFailure(exception); + request.reset(); + performRequest(startTime, hosts, request, responseConsumer, listener); + } + } else { + listener.onDefinitiveFailure(exception); + } + } + + @Override + public void cancelled() { + listener.onDefinitiveFailure(new ExecutionException("request was cancelled", null)); + } + }); } private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) { @@ -247,38 +366,43 @@ public final class RestClient implements Closeable { } /** - * Returns an iterator of hosts to be used for a request call. - * Ideally, the first host is retrieved from the iterator and used successfully for the request. - * Otherwise, after each failure the next host should be retrieved from the iterator so that the request can be retried till - * the iterator is exhausted. The maximum total of attempts is equal to the number of hosts that are available in the iterator. - * The iterator returned will never be empty, rather an {@link IllegalStateException} in case there are no hosts. - * In case there are no healthy hosts available, or dead ones to be be retried, one dead host gets returned. + * Returns an {@link Iterable} of hosts to be used for a request call. + * Ideally, the first host is retrieved from the iterable and used successfully for the request. + * Otherwise, after each failure the next host has to be retrieved from the iterator so that the request can be retried until + * there are no more hosts available to retry against. The maximum total of attempts is equal to the number of hosts in the iterable. + * The iterator returned will never be empty. In case there are no healthy hosts available, or dead ones to be be retried, + * one dead host gets returned so that it can be retried. */ private Iterable nextHost() { - Set filteredHosts = new HashSet<>(hosts); - for (Map.Entry entry : blacklist.entrySet()) { - if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) { - filteredHosts.remove(entry.getKey()); - } - } - - if (filteredHosts.isEmpty()) { - //last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried - List> sortedHosts = new ArrayList<>(blacklist.entrySet()); - Collections.sort(sortedHosts, new Comparator>() { - @Override - public int compare(Map.Entry o1, Map.Entry o2) { - return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); + Collection nextHosts = Collections.emptySet(); + do { + Set filteredHosts = new HashSet<>(hosts); + for (Map.Entry entry : blacklist.entrySet()) { + if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) { + filteredHosts.remove(entry.getKey()); } - }); - HttpHost deadHost = sortedHosts.get(0).getKey(); - logger.trace("resurrecting host [" + deadHost + "]"); - return Collections.singleton(deadHost); - } - - List rotatedHosts = new ArrayList<>(filteredHosts); - Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement()); - return rotatedHosts; + } + if (filteredHosts.isEmpty()) { + //last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried + List> sortedHosts = new ArrayList<>(blacklist.entrySet()); + if (sortedHosts.size() > 0) { + Collections.sort(sortedHosts, new Comparator>() { + @Override + public int compare(Map.Entry o1, Map.Entry o2) { + return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); + } + }); + HttpHost deadHost = sortedHosts.get(0).getKey(); + logger.trace("resurrecting host [" + deadHost + "]"); + nextHosts = Collections.singleton(deadHost); + } + } else { + List rotatedHosts = new ArrayList<>(filteredHosts); + Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement()); + nextHosts = rotatedHosts; + } + } while(nextHosts.isEmpty()); + return nextHosts; } /** @@ -316,7 +440,21 @@ public final class RestClient implements Closeable { client.close(); } - private static IOException addSuppressedException(IOException suppressedException, IOException currentException) { + private static boolean isSuccessfulResponse(String method, int statusCode) { + return statusCode < 300 || (HttpHead.METHOD_NAME.equals(method) && statusCode == 404); + } + + private static boolean isRetryStatus(int statusCode) { + switch(statusCode) { + case 502: + case 503: + case 504: + return true; + } + return false; + } + + private static Exception addSuppressedException(Exception suppressedException, Exception currentException) { if (suppressedException != null) { currentException.addSuppressed(suppressedException); } @@ -373,156 +511,114 @@ public final class RestClient implements Closeable { } /** - * Returns a new {@link Builder} to help with {@link RestClient} creation. + * Listener used in any async call to wrap the provided user listener (or SyncResponseListener in sync calls). + * Allows to track potential failures coming from the different retry attempts and returning to the original listener + * only when we got a response (successful or not to be retried) or there are no hosts to retry against. */ - public static Builder builder(HttpHost... hosts) { - return new Builder(hosts); - } + static class FailureTrackingResponseListener { + private final ResponseListener responseListener; + private volatile Exception exception; - /** - * Rest client builder. Helps creating a new {@link RestClient}. - */ - public static final class Builder { - public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000; - public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 10000; - public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS; - public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS = 500; - - private static final Header[] EMPTY_HEADERS = new Header[0]; - - private final HttpHost[] hosts; - private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS; - private Header[] defaultHeaders = EMPTY_HEADERS; - private FailureListener failureListener; - private HttpClientConfigCallback httpClientConfigCallback; - private RequestConfigCallback requestConfigCallback; - - /** - * Creates a new builder instance and sets the hosts that the client will send requests to. - */ - private Builder(HttpHost... hosts) { - if (hosts == null || hosts.length == 0) { - throw new IllegalArgumentException("no hosts provided"); - } - this.hosts = hosts; + FailureTrackingResponseListener(ResponseListener responseListener) { + this.responseListener = responseListener; } /** - * Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request. - * {@link #DEFAULT_MAX_RETRY_TIMEOUT_MILLIS} if not specified. - * - * @throws IllegalArgumentException if maxRetryTimeoutMillis is not greater than 0 + * Notifies the caller of a response through the wrapped listener */ - public Builder setMaxRetryTimeoutMillis(int maxRetryTimeoutMillis) { - if (maxRetryTimeoutMillis <= 0) { - throw new IllegalArgumentException("maxRetryTimeoutMillis must be greater than 0"); - } - this.maxRetryTimeout = maxRetryTimeoutMillis; - return this; + void onSuccess(Response response) { + responseListener.onSuccess(response); } /** - * Sets the default request headers, to be used sent with every request unless overridden on a per request basis + * Tracks one last definitive failure and returns to the caller by notifying the wrapped listener */ - public Builder setDefaultHeaders(Header[] defaultHeaders) { - Objects.requireNonNull(defaultHeaders, "defaultHeaders must not be null"); - for (Header defaultHeader : defaultHeaders) { - Objects.requireNonNull(defaultHeader, "default header must not be null"); - } - this.defaultHeaders = defaultHeaders; - return this; + void onDefinitiveFailure(Exception exception) { + trackFailure(exception); + responseListener.onFailure(this.exception); } /** - * Sets the {@link FailureListener} to be notified for each request failure + * Tracks an exception, which caused a retry hence we should not return yet to the caller */ - public Builder setFailureListener(FailureListener failureListener) { - Objects.requireNonNull(failureListener, "failureListener must not be null"); - this.failureListener = failureListener; - return this; - } - - /** - * Sets the {@link HttpClientConfigCallback} to be used to customize http client configuration - */ - public Builder setHttpClientConfigCallback(HttpClientConfigCallback httpClientConfigCallback) { - Objects.requireNonNull(httpClientConfigCallback, "httpClientConfigCallback must not be null"); - this.httpClientConfigCallback = httpClientConfigCallback; - return this; - } - - /** - * Sets the {@link RequestConfigCallback} to be used to customize http client configuration - */ - public Builder setRequestConfigCallback(RequestConfigCallback requestConfigCallback) { - Objects.requireNonNull(requestConfigCallback, "requestConfigCallback must not be null"); - this.requestConfigCallback = requestConfigCallback; - return this; - } - - /** - * Creates a new {@link RestClient} based on the provided configuration. - */ - public RestClient build() { - if (failureListener == null) { - failureListener = new FailureListener(); - } - CloseableHttpClient httpClient = createHttpClient(); - return new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, failureListener); - } - - private CloseableHttpClient createHttpClient() { - //default timeouts are all infinite - RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS) - .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT_MILLIS) - .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS); - - if (requestConfigCallback != null) { - requestConfigCallback.customizeRequestConfig(requestConfigBuilder); - } - RequestConfig requestConfig = requestConfigBuilder.build(); - - PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); - //default settings may be too constraining - connectionManager.setDefaultMaxPerRoute(10); - connectionManager.setMaxTotal(30); - - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().setConnectionManager(connectionManager) - .setDefaultRequestConfig(requestConfig); - - if (httpClientConfigCallback != null) { - httpClientConfigCallback.customizeHttpClient(httpClientBuilder); - } - return httpClientBuilder.build(); + void trackFailure(Exception exception) { + this.exception = addSuppressedException(this.exception, exception); } } /** - * Callback used the default {@link RequestConfig} being set to the {@link CloseableHttpClient} - * @see HttpClientBuilder#setDefaultRequestConfig + * Listener used in any sync performRequest calls, it waits for a response or an exception back up to a timeout */ - public interface RequestConfigCallback { - /** - * Allows to customize the {@link RequestConfig} that will be used with each request. - * It is common to customize the different timeout values through this method without losing any other useful default - * value that the {@link RestClient.Builder} internally sets. - */ - void customizeRequestConfig(RequestConfig.Builder requestConfigBuilder); - } + static class SyncResponseListener implements ResponseListener { + private final CountDownLatch latch = new CountDownLatch(1); + private final AtomicReference response = new AtomicReference<>(); + private final AtomicReference exception = new AtomicReference<>(); + + private final long timeout; + + SyncResponseListener(long timeout) { + assert timeout > 0; + this.timeout = timeout; + } + + @Override + public void onSuccess(Response response) { + Objects.requireNonNull(response, "response must not be null"); + boolean wasResponseNull = this.response.compareAndSet(null, response); + if (wasResponseNull == false) { + throw new IllegalStateException("response is already set"); + } + + latch.countDown(); + } + + @Override + public void onFailure(Exception exception) { + Objects.requireNonNull(exception, "exception must not be null"); + boolean wasExceptionNull = this.exception.compareAndSet(null, exception); + if (wasExceptionNull == false) { + throw new IllegalStateException("exception is already set"); + } + latch.countDown(); + } - /** - * Callback used to customize the {@link CloseableHttpClient} instance used by a {@link RestClient} instance. - * Allows to customize default {@link RequestConfig} being set to the client and any parameter that - * can be set through {@link HttpClientBuilder} - */ - public interface HttpClientConfigCallback { /** - * Allows to customize the {@link CloseableHttpClient} being created and used by the {@link RestClient}. - * It is common to customzie the default {@link org.apache.http.client.CredentialsProvider} through this method, - * without losing any other useful default value that the {@link RestClient.Builder} internally sets. - * Also useful to setup ssl through {@link SSLSocketFactoryHttpConfigCallback}. + * Waits (up to a timeout) for some result of the request: either a response, or an exception. */ - void customizeHttpClient(HttpClientBuilder httpClientBuilder); + Response get() throws IOException { + try { + //providing timeout is just a safety measure to prevent everlasting waits + //the different client timeouts should already do their jobs + if (latch.await(timeout, TimeUnit.MILLISECONDS) == false) { + throw new IOException("listener timeout after waiting for [" + timeout + "] ms"); + } + } catch (InterruptedException e) { + throw new RuntimeException("thread waiting for the response was interrupted", e); + } + + Exception exception = this.exception.get(); + Response response = this.response.get(); + if (exception != null) { + if (response != null) { + IllegalStateException e = new IllegalStateException("response and exception are unexpectedly set at the same time"); + e.addSuppressed(exception); + throw e; + } + //try and leave the exception untouched as much as possible but we don't want to just add throws Exception clause everywhere + if (exception instanceof IOException) { + throw (IOException) exception; + } + if (exception instanceof RuntimeException){ + throw (RuntimeException) exception; + } + throw new RuntimeException("error while performing request", exception); + } + + if (response == null) { + throw new IllegalStateException("response not set and no exception caught either"); + } + return response; + } } /** @@ -533,7 +629,7 @@ public final class RestClient implements Closeable { /** * Notifies that the host provided as argument has just failed */ - public void onFailure(HttpHost host) throws IOException { + public void onFailure(HttpHost host) { } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java new file mode 100644 index 00000000000..4d5b72eba49 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java @@ -0,0 +1,179 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.nio.conn.SchemeIOSessionStrategy; + +import java.util.Objects; + +/** + * Helps creating a new {@link RestClient}. Allows to set the most common http client configuration options when internally + * creating the underlying {@link org.apache.http.nio.client.HttpAsyncClient}. Also allows to provide an externally created + * {@link org.apache.http.nio.client.HttpAsyncClient} in case additional customization is needed. + */ +public final class RestClientBuilder { + public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000; + public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 10000; + public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS; + public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS = 500; + public static final int DEFAULT_MAX_CONN_PER_ROUTE = 10; + public static final int DEFAULT_MAX_CONN_TOTAL = 30; + + private static final Header[] EMPTY_HEADERS = new Header[0]; + + private final HttpHost[] hosts; + private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS; + private Header[] defaultHeaders = EMPTY_HEADERS; + private RestClient.FailureListener failureListener; + private HttpClientConfigCallback httpClientConfigCallback; + private RequestConfigCallback requestConfigCallback; + + /** + * Creates a new builder instance and sets the hosts that the client will send requests to. + */ + RestClientBuilder(HttpHost... hosts) { + if (hosts == null || hosts.length == 0) { + throw new IllegalArgumentException("no hosts provided"); + } + for (HttpHost host : hosts) { + Objects.requireNonNull(host, "host cannot be null"); + } + this.hosts = hosts; + } + + /** + * Sets the default request headers, which will be sent along with each request + */ + public RestClientBuilder setDefaultHeaders(Header[] defaultHeaders) { + Objects.requireNonNull(defaultHeaders, "defaultHeaders must not be null"); + for (Header defaultHeader : defaultHeaders) { + Objects.requireNonNull(defaultHeader, "default header must not be null"); + } + this.defaultHeaders = defaultHeaders; + return this; + } + + /** + * Sets the {@link RestClient.FailureListener} to be notified for each request failure + */ + public RestClientBuilder setFailureListener(RestClient.FailureListener failureListener) { + Objects.requireNonNull(failureListener, "failureListener must not be null"); + this.failureListener = failureListener; + return this; + } + + /** + * Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request. + * {@link #DEFAULT_MAX_RETRY_TIMEOUT_MILLIS} if not specified. + * + * @throws IllegalArgumentException if maxRetryTimeoutMillis is not greater than 0 + */ + public RestClientBuilder setMaxRetryTimeoutMillis(int maxRetryTimeoutMillis) { + if (maxRetryTimeoutMillis <= 0) { + throw new IllegalArgumentException("maxRetryTimeoutMillis must be greater than 0"); + } + this.maxRetryTimeout = maxRetryTimeoutMillis; + return this; + } + + /** + * Sets the {@link HttpClientConfigCallback} to be used to customize http client configuration + */ + public RestClientBuilder setHttpClientConfigCallback(HttpClientConfigCallback httpClientConfigCallback) { + Objects.requireNonNull(httpClientConfigCallback, "httpClientConfigCallback must not be null"); + this.httpClientConfigCallback = httpClientConfigCallback; + return this; + } + + /** + * Sets the {@link RequestConfigCallback} to be used to customize http client configuration + */ + public RestClientBuilder setRequestConfigCallback(RequestConfigCallback requestConfigCallback) { + Objects.requireNonNull(requestConfigCallback, "requestConfigCallback must not be null"); + this.requestConfigCallback = requestConfigCallback; + return this; + } + + /** + * Creates a new {@link RestClient} based on the provided configuration. + */ + public RestClient build() { + if (failureListener == null) { + failureListener = new RestClient.FailureListener(); + } + CloseableHttpAsyncClient httpClient = createHttpClient(); + RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, failureListener); + httpClient.start(); + return restClient; + } + + private CloseableHttpAsyncClient createHttpClient() { + //default timeouts are all infinite + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS) + .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT_MILLIS) + .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS); + if (requestConfigCallback != null) { + requestConfigBuilder = requestConfigCallback.customizeRequestConfig(requestConfigBuilder); + } + + HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClientBuilder.create().setDefaultRequestConfig(requestConfigBuilder.build()) + //default settings for connection pooling may be too constraining + .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL); + if (httpClientConfigCallback != null) { + httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder); + } + return httpClientBuilder.build(); + } + + /** + * Callback used the default {@link RequestConfig} being set to the {@link CloseableHttpClient} + * @see HttpClientBuilder#setDefaultRequestConfig + */ + public interface RequestConfigCallback { + /** + * Allows to customize the {@link RequestConfig} that will be used with each request. + * It is common to customize the different timeout values through this method without losing any other useful default + * value that the {@link RestClientBuilder} internally sets. + */ + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder); + } + + /** + * Callback used to customize the {@link CloseableHttpClient} instance used by a {@link RestClient} instance. + * Allows to customize default {@link RequestConfig} being set to the client and any parameter that + * can be set through {@link HttpClientBuilder} + */ + public interface HttpClientConfigCallback { + /** + * Allows to customize the {@link CloseableHttpAsyncClient} being created and used by the {@link RestClient}. + * Commonly used to customize the default {@link org.apache.http.client.CredentialsProvider} for authentication + * or the {@link SchemeIOSessionStrategy} for communication through ssl without losing any other useful default + * value that the {@link RestClientBuilder} internally sets, like connection pooling. + */ + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder); + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/SSLSocketFactoryHttpConfigCallback.java b/client/rest/src/main/java/org/elasticsearch/client/SSLSocketFactoryHttpConfigCallback.java deleted file mode 100644 index 3f18f9938c1..00000000000 --- a/client/rest/src/main/java/org/elasticsearch/client/SSLSocketFactoryHttpConfigCallback.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client; - -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; - -/** - * Helps configuring the http client when needing to communicate over ssl. It effectively replaces the connection manager - * with one that has ssl properly configured thanks to the provided {@link SSLConnectionSocketFactory}. - */ -public class SSLSocketFactoryHttpConfigCallback implements RestClient.HttpClientConfigCallback { - - private final SSLConnectionSocketFactory sslSocketFactory; - - public SSLSocketFactoryHttpConfigCallback(SSLConnectionSocketFactory sslSocketFactory) { - this.sslSocketFactory = sslSocketFactory; - } - - @Override - public void customizeHttpClient(HttpClientBuilder httpClientBuilder) { - Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.getSocketFactory()) - .register("https", sslSocketFactory).build(); - PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - //default settings may be too constraining - connectionManager.setDefaultMaxPerRoute(10); - connectionManager.setMaxTotal(30); - httpClientBuilder.setConnectionManager(connectionManager); - } -} diff --git a/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java b/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java new file mode 100644 index 00000000000..f6ec388d09d --- /dev/null +++ b/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.StatusLine; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.message.BasicStatusLine; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +public class FailureTrackingResponseListenerTests extends RestClientTestCase { + + public void testOnSuccess() { + MockResponseListener responseListener = new MockResponseListener(); + RestClient.FailureTrackingResponseListener listener = new RestClient.FailureTrackingResponseListener(responseListener); + + } + + public void testOnFailure() { + MockResponseListener responseListener = new MockResponseListener(); + RestClient.FailureTrackingResponseListener listener = new RestClient.FailureTrackingResponseListener(responseListener); + int numIters = randomIntBetween(1, 10); + Exception[] expectedExceptions = new Exception[numIters]; + for (int i = 0; i < numIters; i++) { + RuntimeException runtimeException = new RuntimeException("test" + i); + expectedExceptions[i] = runtimeException; + listener.trackFailure(runtimeException); + assertNull(responseListener.response.get()); + assertNull(responseListener.exception.get()); + } + + if (randomBoolean()) { + Response response = mockResponse(); + listener.onSuccess(response); + assertSame(response, responseListener.response.get()); + assertNull(responseListener.exception.get()); + } else { + RuntimeException runtimeException = new RuntimeException("definitive"); + listener.onDefinitiveFailure(runtimeException); + assertNull(responseListener.response.get()); + Throwable exception = responseListener.exception.get(); + assertSame(runtimeException, exception); + + int i = numIters - 1; + do { + assertNotNull(exception.getSuppressed()); + assertEquals(1, exception.getSuppressed().length); + assertSame(expectedExceptions[i--], exception.getSuppressed()[0]); + exception = exception.getSuppressed()[0]; + } while(i >= 0); + } + } + + private static class MockResponseListener implements ResponseListener { + private final AtomicReference response = new AtomicReference<>(); + private final AtomicReference exception = new AtomicReference<>(); + + @Override + public void onSuccess(Response response) { + if (this.response.compareAndSet(null, response) == false) { + throw new IllegalStateException("onSuccess was called multiple times"); + } + } + + @Override + public void onFailure(Exception exception) { + if (this.exception.compareAndSet(null, exception) == false) { + throw new IllegalStateException("onFailure was called multiple times"); + } + } + } + + private static Response mockResponse() { + ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); + RequestLine requestLine = new BasicRequestLine("GET", "/", protocolVersion); + StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK"); + HttpResponse httpResponse = new BasicHttpResponse(statusLine); + return new Response(requestLine, new HttpHost("localhost", 9200), httpResponse); + } +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java b/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java new file mode 100644 index 00000000000..d30a9e00b53 --- /dev/null +++ b/client/rest/src/test/java/org/elasticsearch/client/HeapBufferedAsyncResponseConsumerTests.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.ContentTooLongException; +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.StatusLine; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; +import org.apache.http.nio.ContentDecoder; +import org.apache.http.nio.IOControl; +import org.apache.http.protocol.HttpContext; + +import static org.elasticsearch.client.HeapBufferedAsyncResponseConsumer.DEFAULT_BUFFER_LIMIT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class HeapBufferedAsyncResponseConsumerTests extends RestClientTestCase { + + //maximum buffer that this test ends up allocating is 50MB + private static final int MAX_TEST_BUFFER_SIZE = 50 * 1024 * 1024; + + public void testResponseProcessing() throws Exception { + ContentDecoder contentDecoder = mock(ContentDecoder.class); + IOControl ioControl = mock(IOControl.class); + HttpContext httpContext = mock(HttpContext.class); + + HeapBufferedAsyncResponseConsumer consumer = spy(new HeapBufferedAsyncResponseConsumer()); + + ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); + StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK"); + HttpResponse httpResponse = new BasicHttpResponse(statusLine); + httpResponse.setEntity(new StringEntity("test")); + + //everything goes well + consumer.responseReceived(httpResponse); + consumer.consumeContent(contentDecoder, ioControl); + consumer.responseCompleted(httpContext); + + verify(consumer).releaseResources(); + verify(consumer).buildResult(httpContext); + assertTrue(consumer.isDone()); + assertSame(httpResponse, consumer.getResult()); + + consumer.responseCompleted(httpContext); + verify(consumer, times(1)).releaseResources(); + verify(consumer, times(1)).buildResult(httpContext); + } + + public void testDefaultBufferLimit() throws Exception { + HeapBufferedAsyncResponseConsumer consumer = new HeapBufferedAsyncResponseConsumer(); + bufferLimitTest(consumer, DEFAULT_BUFFER_LIMIT); + } + + public void testConfiguredBufferLimit() throws Exception { + try { + new HeapBufferedAsyncResponseConsumer(randomIntBetween(Integer.MIN_VALUE, 0)); + } catch(IllegalArgumentException e) { + assertEquals("bufferLimit must be greater than 0", e.getMessage()); + } + try { + new HeapBufferedAsyncResponseConsumer(0); + } catch(IllegalArgumentException e) { + assertEquals("bufferLimit must be greater than 0", e.getMessage()); + } + int bufferLimit = randomIntBetween(1, MAX_TEST_BUFFER_SIZE - 100); + HeapBufferedAsyncResponseConsumer consumer = new HeapBufferedAsyncResponseConsumer(bufferLimit); + bufferLimitTest(consumer, bufferLimit); + } + + private static void bufferLimitTest(HeapBufferedAsyncResponseConsumer consumer, int bufferLimit) throws Exception { + ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); + StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK"); + consumer.onResponseReceived(new BasicHttpResponse(statusLine)); + + BasicHttpEntity entity = new BasicHttpEntity(); + entity.setContentLength(randomInt(bufferLimit)); + consumer.onEntityEnclosed(entity, ContentType.APPLICATION_JSON); + + entity.setContentLength(randomIntBetween(bufferLimit + 1, MAX_TEST_BUFFER_SIZE)); + try { + consumer.onEntityEnclosed(entity, ContentType.APPLICATION_JSON); + } catch(ContentTooLongException e) { + assertEquals("entity content is too long [" + entity.getContentLength() + + "] for the configured buffer limit [" + bufferLimit + "]", e.getMessage()); + } + } +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/TrackingFailureListener.java b/client/rest/src/test/java/org/elasticsearch/client/HostsTrackingFailureListener.java similarity index 85% rename from client/rest/src/test/java/org/elasticsearch/client/TrackingFailureListener.java rename to client/rest/src/test/java/org/elasticsearch/client/HostsTrackingFailureListener.java index 35842823923..e2f0ba81f6e 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/TrackingFailureListener.java +++ b/client/rest/src/test/java/org/elasticsearch/client/HostsTrackingFailureListener.java @@ -21,7 +21,6 @@ package org.elasticsearch.client; import org.apache.http.HttpHost; -import java.io.IOException; import java.util.HashSet; import java.util.Set; @@ -30,13 +29,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; /** - * {@link org.elasticsearch.client.RestClient.FailureListener} impl that allows to track when it gets called + * {@link org.elasticsearch.client.RestClient.FailureListener} impl that allows to track when it gets called for which host. */ -class TrackingFailureListener extends RestClient.FailureListener { - private Set hosts = new HashSet<>(); +class HostsTrackingFailureListener extends RestClient.FailureListener { + private volatile Set hosts = new HashSet<>(); @Override - public void onFailure(HttpHost host) throws IOException { + public void onFailure(HttpHost host) { hosts.add(host); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java index 4d3ad75b5e8..f00060a02de 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java @@ -35,6 +35,8 @@ import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; +import org.apache.http.nio.entity.NByteArrayEntity; +import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; import java.io.ByteArrayInputStream; @@ -97,14 +99,24 @@ public class RequestLoggerTests extends RestClientTestCase { expected += " -d '" + requestBody + "'"; HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request; HttpEntity entity; - if (getRandom().nextBoolean()) { - entity = new StringEntity(requestBody, StandardCharsets.UTF_8); - } else { - entity = new InputStreamEntity(new ByteArrayInputStream(requestBody.getBytes(StandardCharsets.UTF_8))); + switch(RandomInts.randomIntBetween(getRandom(), 0, 3)) { + case 0: + entity = new StringEntity(requestBody, StandardCharsets.UTF_8); + break; + case 1: + entity = new InputStreamEntity(new ByteArrayInputStream(requestBody.getBytes(StandardCharsets.UTF_8))); + break; + case 2: + entity = new NStringEntity(requestBody, StandardCharsets.UTF_8); + break; + case 3: + entity = new NByteArrayEntity(requestBody.getBytes(StandardCharsets.UTF_8)); + break; + default: + throw new UnsupportedOperationException(); } enclosingRequest.setEntity(entity); } - String traceRequest = RequestLogger.buildTraceRequest(request, host); assertThat(traceRequest, equalTo(expected)); if (hasBody) { diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java index f032933db24..ca671862124 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java @@ -23,7 +23,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHeader; import java.io.IOException; @@ -50,12 +50,16 @@ public class RestClientBuilderTests extends RestClientTestCase { } try { - RestClient.builder(new HttpHost[]{new HttpHost("localhost", 9200), null}).build(); + RestClient.builder(new HttpHost("localhost", 9200), null); fail("should have failed"); } catch(NullPointerException e) { assertEquals("host cannot be null", e.getMessage()); } + try (RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build()) { + assertNotNull(restClient); + } + try { RestClient.builder(new HttpHost("localhost", 9200)) .setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0)); @@ -104,18 +108,20 @@ public class RestClientBuilderTests extends RestClientTestCase { for (int i = 0; i < numNodes; i++) { hosts[i] = new HttpHost("localhost", 9200 + i); } - RestClient.Builder builder = RestClient.builder(hosts); + RestClientBuilder builder = RestClient.builder(hosts); if (getRandom().nextBoolean()) { - builder.setHttpClientConfigCallback(new RestClient.HttpClientConfigCallback() { + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { @Override - public void customizeHttpClient(HttpClientBuilder httpClientBuilder) { + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + return httpClientBuilder; } }); } if (getRandom().nextBoolean()) { - builder.setRequestConfigCallback(new RestClient.RequestConfigCallback() { + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { @Override - public void customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + return requestConfigBuilder; } }); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java index 4a14c174353..455affea9d5 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java @@ -27,6 +27,7 @@ import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; import org.apache.http.Consts; import org.apache.http.Header; +import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; @@ -47,6 +48,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes; import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods; @@ -140,10 +144,10 @@ public class RestClientIntegTests extends RestClientTestCase { * to set/add headers to the {@link org.apache.http.client.HttpClient}. * Exercises the test http server ability to send back whatever headers it received. */ - public void testHeaders() throws Exception { + public void testHeaders() throws IOException { for (String method : getHttpMethods()) { Set standardHeaders = new HashSet<>( - Arrays.asList("Accept-encoding", "Connection", "Host", "User-agent", "Date")); + Arrays.asList("Connection", "Host", "User-agent", "Date")); if (method.equals("HEAD") == false) { standardHeaders.add("Content-length"); } @@ -162,9 +166,9 @@ public class RestClientIntegTests extends RestClientTestCase { int statusCode = randomStatusCode(getRandom()); Response esResponse; - try (Response response = restClient.performRequest(method, "/" + statusCode, - Collections.emptyMap(), null, headers)) { - esResponse = response; + try { + esResponse = restClient.performRequest(method, "/" + statusCode, Collections.emptyMap(), + (HttpEntity)null, headers); } catch(ResponseException e) { esResponse = e.getResponse(); } @@ -188,7 +192,7 @@ public class RestClientIntegTests extends RestClientTestCase { * out of the box by {@link org.apache.http.client.HttpClient}. * Exercises the test http server ability to send back whatever body it received. */ - public void testDeleteWithBody() throws Exception { + public void testDeleteWithBody() throws IOException { bodyTest("DELETE"); } @@ -197,25 +201,74 @@ public class RestClientIntegTests extends RestClientTestCase { * out of the box by {@link org.apache.http.client.HttpClient}. * Exercises the test http server ability to send back whatever body it received. */ - public void testGetWithBody() throws Exception { + public void testGetWithBody() throws IOException { bodyTest("GET"); } - private void bodyTest(String method) throws Exception { + private void bodyTest(String method) throws IOException { String requestBody = "{ \"field\": \"value\" }"; StringEntity entity = new StringEntity(requestBody); - Response esResponse; - String responseBody; int statusCode = randomStatusCode(getRandom()); - try (Response response = restClient.performRequest(method, "/" + statusCode, - Collections.emptyMap(), entity)) { - responseBody = EntityUtils.toString(response.getEntity()); - esResponse = response; + Response esResponse; + try { + esResponse = restClient.performRequest(method, "/" + statusCode, Collections.emptyMap(), entity); } catch(ResponseException e) { - responseBody = e.getResponseBody(); esResponse = e.getResponse(); } assertEquals(statusCode, esResponse.getStatusLine().getStatusCode()); - assertEquals(requestBody, responseBody); + assertEquals(requestBody, EntityUtils.toString(esResponse.getEntity())); + } + + public void testAsyncRequests() throws Exception { + int numRequests = randomIntBetween(5, 20); + final CountDownLatch latch = new CountDownLatch(numRequests); + final List responses = new CopyOnWriteArrayList<>(); + for (int i = 0; i < numRequests; i++) { + final String method = RestClientTestUtil.randomHttpMethod(getRandom()); + final int statusCode = randomStatusCode(getRandom()); + restClient.performRequest(method, "/" + statusCode, new ResponseListener() { + @Override + public void onSuccess(Response response) { + responses.add(new TestResponse(method, statusCode, response)); + latch.countDown(); + } + + @Override + public void onFailure(Exception exception) { + responses.add(new TestResponse(method, statusCode, exception)); + latch.countDown(); + } + }); + } + assertTrue(latch.await(5, TimeUnit.SECONDS)); + + assertEquals(numRequests, responses.size()); + for (TestResponse response : responses) { + assertEquals(response.method, response.getResponse().getRequestLine().getMethod()); + assertEquals(response.statusCode, response.getResponse().getStatusLine().getStatusCode()); + + } + } + + private static class TestResponse { + private final String method; + private final int statusCode; + private final Object response; + + TestResponse(String method, int statusCode, Object response) { + this.method = method; + this.statusCode = statusCode; + this.response = response; + } + + Response getResponse() { + if (response instanceof Response) { + return (Response) response; + } + if (response instanceof ResponseException) { + return ((ResponseException) response).getResponse(); + } + throw new AssertionError("unexpected response " + response.getClass()); + } } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 1a469f2e2ca..2a2d279689a 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -22,14 +22,17 @@ package org.elasticsearch.client; import com.carrotsearch.randomizedtesting.generators.RandomInts; import org.apache.http.Header; import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; import org.apache.http.ProtocolVersion; import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.concurrent.FutureCallback; import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; +import org.apache.http.nio.protocol.HttpAsyncRequestProducer; +import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import org.junit.Before; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -39,6 +42,7 @@ import java.net.SocketTimeoutException; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.Future; import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode; import static org.elasticsearch.client.RestClientTestUtil.randomErrorRetryStatusCode; @@ -62,57 +66,61 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { private RestClient restClient; private HttpHost[] httpHosts; - private TrackingFailureListener failureListener; + private HostsTrackingFailureListener failureListener; @Before + @SuppressWarnings("unchecked") public void createRestClient() throws IOException { - CloseableHttpClient httpClient = mock(CloseableHttpClient.class); - when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenAnswer(new Answer() { + CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class); + when(httpClient.execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class), + any(FutureCallback.class))).thenAnswer(new Answer>() { @Override - public CloseableHttpResponse answer(InvocationOnMock invocationOnMock) throws Throwable { - HttpHost httpHost = (HttpHost) invocationOnMock.getArguments()[0]; - HttpUriRequest request = (HttpUriRequest) invocationOnMock.getArguments()[1]; + public Future answer(InvocationOnMock invocationOnMock) throws Throwable { + HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0]; + HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest(); + HttpHost httpHost = requestProducer.getTarget(); + FutureCallback futureCallback = (FutureCallback) invocationOnMock.getArguments()[2]; //return the desired status code or exception depending on the path if (request.getURI().getPath().equals("/soe")) { - throw new SocketTimeoutException(httpHost.toString()); + futureCallback.failed(new SocketTimeoutException(httpHost.toString())); } else if (request.getURI().getPath().equals("/coe")) { - throw new ConnectTimeoutException(httpHost.toString()); + futureCallback.failed(new ConnectTimeoutException(httpHost.toString())); } else if (request.getURI().getPath().equals("/ioe")) { - throw new IOException(httpHost.toString()); + futureCallback.failed(new IOException(httpHost.toString())); + } else { + int statusCode = Integer.parseInt(request.getURI().getPath().substring(1)); + StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, ""); + futureCallback.completed(new BasicHttpResponse(statusLine)); } - int statusCode = Integer.parseInt(request.getURI().getPath().substring(1)); - StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, ""); - return new CloseableBasicHttpResponse(statusLine); + return null; } }); - int numHosts = RandomInts.randomIntBetween(getRandom(), 2, 5); httpHosts = new HttpHost[numHosts]; for (int i = 0; i < numHosts; i++) { httpHosts[i] = new HttpHost("localhost", 9200 + i); } - failureListener = new TrackingFailureListener(); + failureListener = new HostsTrackingFailureListener(); restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, failureListener); } - public void testRoundRobinOkStatusCodes() throws Exception { + public void testRoundRobinOkStatusCodes() throws IOException { int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5); for (int i = 0; i < numIters; i++) { Set hostsSet = new HashSet<>(); Collections.addAll(hostsSet, httpHosts); for (int j = 0; j < httpHosts.length; j++) { int statusCode = randomOkStatusCode(getRandom()); - try (Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode)) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode)); - assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost())); - } + Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode); + assertEquals(statusCode, response.getStatusLine().getStatusCode()); + assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost())); } assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size()); } failureListener.assertNotCalled(); } - public void testRoundRobinNoRetryErrors() throws Exception { + public void testRoundRobinNoRetryErrors() throws IOException { int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5); for (int i = 0; i < numIters; i++) { Set hostsSet = new HashSet<>(); @@ -120,11 +128,12 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { for (int j = 0; j < httpHosts.length; j++) { String method = randomHttpMethod(getRandom()); int statusCode = randomErrorNoRetryStatusCode(getRandom()); - try (Response response = restClient.performRequest(method, "/" + statusCode)) { + try { + Response response = restClient.performRequest(method, "/" + statusCode); if (method.equals("HEAD") && statusCode == 404) { //no exception gets thrown although we got a 404 - assertThat(response.getStatusLine().getStatusCode(), equalTo(404)); - assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode)); + assertEquals(404, response.getStatusLine().getStatusCode()); + assertEquals(statusCode, response.getStatusLine().getStatusCode()); assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost())); } else { fail("request should have failed"); @@ -134,7 +143,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { throw e; } Response response = e.getResponse(); - assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode)); + assertEquals(statusCode, response.getStatusLine().getStatusCode()); assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost())); assertEquals(0, e.getSuppressed().length); } @@ -144,7 +153,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { failureListener.assertNotCalled(); } - public void testRoundRobinRetryErrors() throws Exception { + public void testRoundRobinRetryErrors() throws IOException { String retryEndpoint = randomErrorRetryEndpoint(); try { restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint); @@ -156,7 +165,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { failureListener.assertCalled(httpHosts); do { Response response = e.getResponse(); - assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1)))); + assertEquals(Integer.parseInt(retryEndpoint.substring(1)), response.getStatusLine().getStatusCode()); assertTrue("host [" + response.getHost() + "] not found, most likely used multiple times", hostsSet.remove(response.getHost())); if (e.getSuppressed().length > 0) { @@ -223,8 +232,8 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { for (int y = 0; y < iters; y++) { int statusCode = randomErrorNoRetryStatusCode(getRandom()); Response response; - try (Response esResponse = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode)) { - response = esResponse; + try { + response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode); } catch(ResponseException e) { response = e.getResponse(); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index e7c9de934b6..e347dfecc12 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -26,9 +26,9 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; import org.apache.http.ProtocolVersion; import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpOptions; import org.apache.http.client.methods.HttpPatch; @@ -37,11 +37,15 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpTrace; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.concurrent.FutureCallback; import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.message.BasicHeader; +import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; +import org.apache.http.nio.protocol.HttpAsyncRequestProducer; +import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import org.apache.http.util.EntityUtils; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -51,11 +55,11 @@ import org.mockito.stubbing.Answer; import java.io.IOException; import java.net.SocketTimeoutException; import java.net.URI; -import java.net.URISyntaxException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.Future; import static org.elasticsearch.client.RestClientTestUtil.getAllErrorStatusCodes; import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods; @@ -86,39 +90,48 @@ public class RestClientSingleHostTests extends RestClientTestCase { private RestClient restClient; private Header[] defaultHeaders; private HttpHost httpHost; - private CloseableHttpClient httpClient; - private TrackingFailureListener failureListener; + private CloseableHttpAsyncClient httpClient; + private HostsTrackingFailureListener failureListener; @Before + @SuppressWarnings("unchecked") public void createRestClient() throws IOException { - httpClient = mock(CloseableHttpClient.class); - when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenAnswer(new Answer() { - @Override - public CloseableHttpResponse answer(InvocationOnMock invocationOnMock) throws Throwable { - HttpUriRequest request = (HttpUriRequest) invocationOnMock.getArguments()[1]; - //return the desired status code or exception depending on the path - if (request.getURI().getPath().equals("/soe")) { - throw new SocketTimeoutException(); - } else if (request.getURI().getPath().equals("/coe")) { - throw new ConnectTimeoutException(); - } - int statusCode = Integer.parseInt(request.getURI().getPath().substring(1)); - StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, ""); + httpClient = mock(CloseableHttpAsyncClient.class); + when(httpClient.execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class), + any(FutureCallback.class))).thenAnswer(new Answer>() { + @Override + public Future answer(InvocationOnMock invocationOnMock) throws Throwable { + HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0]; + FutureCallback futureCallback = (FutureCallback) invocationOnMock.getArguments()[2]; + HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest(); + //return the desired status code or exception depending on the path + if (request.getURI().getPath().equals("/soe")) { + futureCallback.failed(new SocketTimeoutException()); + } else if (request.getURI().getPath().equals("/coe")) { + futureCallback.failed(new ConnectTimeoutException()); + } else { + int statusCode = Integer.parseInt(request.getURI().getPath().substring(1)); + StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, ""); - CloseableHttpResponse httpResponse = new CloseableBasicHttpResponse(statusLine); - //return the same body that was sent - if (request instanceof HttpEntityEnclosingRequest) { - HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity(); - if (entity != null) { - assertTrue("the entity is not repeatable, cannot set it to the response directly", entity.isRepeatable()); - httpResponse.setEntity(entity); + HttpResponse httpResponse = new BasicHttpResponse(statusLine); + //return the same body that was sent + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity(); + if (entity != null) { + assertTrue("the entity is not repeatable, cannot set it to the response directly", + entity.isRepeatable()); + httpResponse.setEntity(entity); + } + } + //return the same headers that were sent + httpResponse.setHeaders(request.getAllHeaders()); + futureCallback.completed(httpResponse); + } + return null; } - } - //return the same headers that were sent - httpResponse.setHeaders(request.getAllHeaders()); - return httpResponse; - } - }); + }); + + int numHeaders = RandomInts.randomIntBetween(getRandom(), 0, 3); defaultHeaders = new Header[numHeaders]; for (int i = 0; i < numHeaders; i++) { @@ -127,20 +140,22 @@ public class RestClientSingleHostTests extends RestClientTestCase { defaultHeaders[i] = new BasicHeader(headerName, headerValue); } httpHost = new HttpHost("localhost", 9200); - failureListener = new TrackingFailureListener(); + failureListener = new HostsTrackingFailureListener(); restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, failureListener); } /** * Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client */ + @SuppressWarnings("unchecked") public void testInternalHttpRequest() throws Exception { - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(HttpUriRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(HttpAsyncRequestProducer.class); int times = 0; for (String httpMethod : getHttpMethods()) { HttpUriRequest expectedRequest = performRandomRequest(httpMethod); - verify(httpClient, times(++times)).execute(any(HttpHost.class), requestArgumentCaptor.capture()); - HttpUriRequest actualRequest = requestArgumentCaptor.getValue(); + verify(httpClient, times(++times)).execute(requestArgumentCaptor.capture(), + any(HttpAsyncResponseConsumer.class), any(FutureCallback.class)); + HttpUriRequest actualRequest = (HttpUriRequest)requestArgumentCaptor.getValue().generateRequest(); assertEquals(expectedRequest.getURI(), actualRequest.getURI()); assertEquals(expectedRequest.getClass(), actualRequest.getClass()); assertArrayEquals(expectedRequest.getAllHeaders(), actualRequest.getAllHeaders()); @@ -184,7 +199,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { /** * End to end test for ok status codes */ - public void testOkStatusCodes() throws Exception { + public void testOkStatusCodes() throws IOException { for (String method : getHttpMethods()) { for (int okStatusCode : getOkStatusCodes()) { Response response = performRequest(method, "/" + okStatusCode); @@ -197,11 +212,12 @@ public class RestClientSingleHostTests extends RestClientTestCase { /** * End to end test for error status codes: they should cause an exception to be thrown, apart from 404 with HEAD requests */ - public void testErrorStatusCodes() throws Exception { + public void testErrorStatusCodes() throws IOException { for (String method : getHttpMethods()) { //error status codes should cause an exception to be thrown for (int errorStatusCode : getAllErrorStatusCodes()) { - try (Response response = performRequest(method, "/" + errorStatusCode)) { + try { + Response response = performRequest(method, "/" + errorStatusCode); if (method.equals("HEAD") && errorStatusCode == 404) { //no exception gets thrown although we got a 404 assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode)); @@ -247,16 +263,14 @@ public class RestClientSingleHostTests extends RestClientTestCase { * End to end test for request and response body. Exercises the mock http client ability to send back * whatever body it has received. */ - public void testBody() throws Exception { + public void testBody() throws IOException { String body = "{ \"field\": \"value\" }"; StringEntity entity = new StringEntity(body); for (String method : Arrays.asList("DELETE", "GET", "PATCH", "POST", "PUT")) { for (int okStatusCode : getOkStatusCodes()) { - try (Response response = restClient.performRequest(method, "/" + okStatusCode, - Collections.emptyMap(), entity)) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode)); - assertThat(EntityUtils.toString(response.getEntity()), equalTo(body)); - } + Response response = restClient.performRequest(method, "/" + okStatusCode, Collections.emptyMap(), entity); + assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode)); + assertThat(EntityUtils.toString(response.getEntity()), equalTo(body)); } for (int errorStatusCode : getAllErrorStatusCodes()) { try { @@ -279,7 +293,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { } } - public void testNullHeaders() throws Exception { + public void testNullHeaders() throws IOException { String method = randomHttpMethod(getRandom()); int statusCode = randomStatusCode(getRandom()); try { @@ -296,7 +310,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { } } - public void testNullParams() throws Exception { + public void testNullParams() throws IOException { String method = randomHttpMethod(getRandom()); int statusCode = randomStatusCode(getRandom()); try { @@ -317,7 +331,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { * End to end test for request and response headers. Exercises the mock http client ability to send back * whatever headers it has received. */ - public void testHeaders() throws Exception { + public void testHeaders() throws IOException { for (String method : getHttpMethods()) { Map expectedHeaders = new HashMap<>(); for (Header defaultHeader : defaultHeaders) { @@ -334,9 +348,8 @@ public class RestClientSingleHostTests extends RestClientTestCase { int statusCode = randomStatusCode(getRandom()); Response esResponse; - try (Response response = restClient.performRequest(method, "/" + statusCode, - Collections.emptyMap(), null, headers)) { - esResponse = response; + try { + esResponse = restClient.performRequest(method, "/" + statusCode, headers); } catch(ResponseException e) { esResponse = e.getResponse(); } @@ -349,7 +362,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { } } - private HttpUriRequest performRandomRequest(String method) throws IOException, URISyntaxException { + private HttpUriRequest performRandomRequest(String method) throws Exception { String uriAsString = "/" + randomStatusCode(getRandom()); URIBuilder uriBuilder = new URIBuilder(uriAsString); Map params = Collections.emptyMap(); @@ -441,7 +454,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { case 1: return restClient.performRequest(method, endpoint, Collections.emptyMap(), headers); case 2: - return restClient.performRequest(method, endpoint, Collections.emptyMap(), null, headers); + return restClient.performRequest(method, endpoint, Collections.emptyMap(), (HttpEntity)null, headers); default: throw new UnsupportedOperationException(); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/SyncResponseListenerTests.java b/client/rest/src/test/java/org/elasticsearch/client/SyncResponseListenerTests.java new file mode 100644 index 00000000000..154efb4cac3 --- /dev/null +++ b/client/rest/src/test/java/org/elasticsearch/client/SyncResponseListenerTests.java @@ -0,0 +1,172 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.StatusLine; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.message.BasicStatusLine; + +import java.io.IOException; +import java.net.URISyntaxException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +public class SyncResponseListenerTests extends RestClientTestCase { + + public void testOnSuccessNullResponse() { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + try { + syncResponseListener.onSuccess(null); + fail("onSuccess should have failed"); + } catch(NullPointerException e) { + assertEquals("response must not be null", e.getMessage()); + } + } + + public void testOnFailureNullException() { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + try { + syncResponseListener.onFailure(null); + fail("onFailure should have failed"); + } catch(NullPointerException e) { + assertEquals("exception must not be null", e.getMessage()); + } + } + + public void testOnSuccess() throws Exception { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + Response mockResponse = mockResponse(); + syncResponseListener.onSuccess(mockResponse); + Response response = syncResponseListener.get(); + assertSame(response, mockResponse); + + try { + syncResponseListener.onSuccess(mockResponse); + fail("get should have failed"); + } catch(IllegalStateException e) { + assertEquals(e.getMessage(), "response is already set"); + } + response = syncResponseListener.get(); + assertSame(response, mockResponse); + + RuntimeException runtimeException = new RuntimeException("test"); + syncResponseListener.onFailure(runtimeException); + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(IllegalStateException e) { + assertEquals("response and exception are unexpectedly set at the same time", e.getMessage()); + assertNotNull(e.getSuppressed()); + assertEquals(1, e.getSuppressed().length); + assertSame(runtimeException, e.getSuppressed()[0]); + } + } + + public void testOnFailure() throws Exception { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + RuntimeException firstException = new RuntimeException("first-test"); + syncResponseListener.onFailure(firstException); + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(RuntimeException e) { + assertSame(firstException, e); + } + + RuntimeException secondException = new RuntimeException("second-test"); + try { + syncResponseListener.onFailure(secondException); + } catch(IllegalStateException e) { + assertEquals(e.getMessage(), "exception is already set"); + } + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(RuntimeException e) { + assertSame(firstException, e); + } + + Response response = mockResponse(); + syncResponseListener.onSuccess(response); + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(IllegalStateException e) { + assertEquals("response and exception are unexpectedly set at the same time", e.getMessage()); + assertNotNull(e.getSuppressed()); + assertEquals(1, e.getSuppressed().length); + assertSame(firstException, e.getSuppressed()[0]); + } + } + + public void testRuntimeExceptionIsNotWrapped() throws Exception { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + RuntimeException runtimeException = new RuntimeException(); + syncResponseListener.onFailure(runtimeException); + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(RuntimeException e) { + assertSame(runtimeException, e); + } + } + + public void testIOExceptionIsNotWrapped() throws Exception { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + IOException ioException = new IOException(); + syncResponseListener.onFailure(ioException); + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(IOException e) { + assertSame(ioException, e); + } + } + + public void testExceptionIsWrapped() throws Exception { + RestClient.SyncResponseListener syncResponseListener = new RestClient.SyncResponseListener(10000); + //we just need any checked exception + URISyntaxException exception = new URISyntaxException("test", "test"); + syncResponseListener.onFailure(exception); + try { + syncResponseListener.get(); + fail("get should have failed"); + } catch(RuntimeException e) { + assertEquals("error while performing request", e.getMessage()); + assertSame(exception, e.getCause()); + } + } + + private static Response mockResponse() { + ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); + RequestLine requestLine = new BasicRequestLine("GET", "/", protocolVersion); + StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "OK"); + HttpResponse httpResponse = new BasicHttpResponse(statusLine); + return new Response(requestLine, new HttpHost("localhost", 9200), httpResponse); + } +} diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 7cf16ee85d8..639871cccb3 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -58,6 +58,11 @@ forbiddenApisTest { signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] } +dependencyLicenses { + mapping from: /http.*/, to: 'httpclient' + mapping from: /commons-.*/, to: 'commons' +} + //JarHell is part of es core, which we don't want to pull in jarHell.enabled=false diff --git a/client/rest/licenses/commons-logging-LICENSE.txt b/client/sniffer/licenses/commons-LICENSE.txt similarity index 100% rename from client/rest/licenses/commons-logging-LICENSE.txt rename to client/sniffer/licenses/commons-LICENSE.txt diff --git a/client/sniffer/licenses/commons-codec-NOTICE.txt b/client/sniffer/licenses/commons-NOTICE.txt similarity index 100% rename from client/sniffer/licenses/commons-codec-NOTICE.txt rename to client/sniffer/licenses/commons-NOTICE.txt diff --git a/client/sniffer/licenses/commons-logging-NOTICE.txt b/client/sniffer/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 556bd03951d..00000000000 --- a/client/sniffer/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - diff --git a/client/sniffer/licenses/httpcore-4.4.4.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/client/sniffer/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore-LICENSE.txt b/client/sniffer/licenses/httpcore-LICENSE.txt deleted file mode 100644 index 32f01eda18f..00000000000 --- a/client/sniffer/licenses/httpcore-LICENSE.txt +++ /dev/null @@ -1,558 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -========================================================================= - -This project includes Public Suffix List copied from - -licensed under the terms of the Mozilla Public License, v. 2.0 - -Full license text: - -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. diff --git a/client/sniffer/licenses/httpcore-NOTICE.txt b/client/sniffer/licenses/httpcore-NOTICE.txt deleted file mode 100644 index 91e5c40c4c6..00000000000 --- a/client/sniffer/licenses/httpcore-NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache HttpComponents Client -Copyright 1999-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java index bfe21f5e7d1..d5b2c67ee02 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java @@ -62,9 +62,8 @@ public class HostsSniffer { * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts */ public List sniffHosts() throws IOException { - try (Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams)) { - return readHosts(response.getEntity()); - } + Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams); + return readHosts(response.getEntity()); } private List readHosts(HttpEntity entity) throws IOException { diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java index 76350057141..cbc77351de9 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java @@ -22,7 +22,6 @@ package org.elasticsearch.client.sniff; import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; -import java.io.IOException; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -55,7 +54,7 @@ public class SniffOnFailureListener extends RestClient.FailureListener { } @Override - public void onFailure(HttpHost host) throws IOException { + public void onFailure(HttpHost host) { if (sniffer == null) { throw new IllegalStateException("sniffer was not set, unable to sniff on failure"); } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java index 74a28cdd222..61fa32e7b62 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import java.io.Closeable; import java.io.IOException; @@ -39,7 +40,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * Must be created via {@link Builder}, which allows to set all of the different options or rely on defaults. * A background task fetches the nodes through the {@link HostsSniffer} and sets them to the {@link RestClient} instance. * It is possible to perform sniffing on failure by creating a {@link SniffOnFailureListener} and providing it as an argument to - * {@link org.elasticsearch.client.RestClient.Builder#setFailureListener(RestClient.FailureListener)}. The Sniffer implementation + * {@link RestClientBuilder#setFailureListener(RestClient.FailureListener)}. The Sniffer implementation * needs to be lazily set to the previously created SniffOnFailureListener through {@link SniffOnFailureListener#setSniffer(Sniffer)}. */ public final class Sniffer implements Closeable { diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java index 6e0c3a728d5..bb375834097 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java @@ -43,7 +43,6 @@ import java.io.OutputStream; import java.io.StringWriter; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -86,7 +85,7 @@ public class HostsSnifferTests extends RestClientTestCase { httpServer.stop(0); } - public void testSniffNodes() throws IOException, URISyntaxException { + public void testSniffNodes() throws IOException { HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()); try (RestClient restClient = RestClient.builder(httpHost).build()) { HostsSniffer.Builder builder = HostsSniffer.builder(restClient).setSniffRequestTimeoutMillis(sniffRequestTimeout); diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java index 6a71d72f60e..bbb1de35663 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java @@ -45,16 +45,17 @@ public class SniffOnFailureListenerTests extends RestClientTestCase { assertEquals("sniffer must not be null", e.getMessage()); } - RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build(); - try (Sniffer sniffer = Sniffer.builder(restClient, new MockHostsSniffer()).build()) { - listener.setSniffer(sniffer); - try { + try (RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build()) { + try (Sniffer sniffer = Sniffer.builder(restClient, new MockHostsSniffer()).build()) { listener.setSniffer(sniffer); - fail("should have failed"); - } catch(IllegalStateException e) { - assertEquals("sniffer can only be set once", e.getMessage()); + try { + listener.setSniffer(sniffer); + fail("should have failed"); + } catch(IllegalStateException e) { + assertEquals("sniffer can only be set once", e.getMessage()); + } + listener.onFailure(new HttpHost("localhost", 9200)); } - listener.onFailure(new HttpHost("localhost", 9200)); } } } diff --git a/client/transport/build.gradle b/client/transport/build.gradle index 3c4f967f766..3573467b0ef 100644 --- a/client/transport/build.gradle +++ b/client/transport/build.gradle @@ -26,6 +26,7 @@ group = 'org.elasticsearch.client' dependencies { compile "org.elasticsearch:elasticsearch:${version}" compile project(path: ':modules:transport-netty3', configuration: 'runtime') + compile project(path: ':modules:transport-netty4', configuration: 'runtime') compile project(path: ':modules:reindex', configuration: 'runtime') compile project(path: ':modules:lang-mustache', configuration: 'runtime') compile project(path: ':modules:percolator', configuration: 'runtime') diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java index 30d4beddae2..aebcf5fa3a3 100644 --- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java +++ b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java @@ -16,30 +16,47 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.transport.client; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.percolator.PercolatorPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.transport.Netty3Plugin; +import org.elasticsearch.transport.Netty4Plugin; import java.util.Arrays; import java.util.Collection; import java.util.Collections; - +import java.util.List; /** * A builder to create an instance of {@link TransportClient} - * This class pre-installs the {@link Netty3Plugin}, {@link ReindexPlugin}, {@link PercolatorPlugin}, and {@link MustachePlugin} + * This class pre-installs the + * {@link Netty3Plugin}, + * {@link Netty4Plugin}, + * {@link ReindexPlugin}, + * {@link PercolatorPlugin}, + * and {@link MustachePlugin} * for the client. These plugins are all elasticsearch core modules required. */ @SuppressWarnings({"unchecked","varargs"}) public class PreBuiltTransportClient extends TransportClient { - private static final Collection> PRE_INSTALLED_PLUGINS = Collections.unmodifiableList(Arrays.asList( - TransportPlugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class)); + + private static final Collection> PRE_INSTALLED_PLUGINS = + Collections.unmodifiableList( + Arrays.asList( + Netty3Plugin.class, + Netty4Plugin.class, + TransportPlugin.class, + ReindexPlugin.class, + PercolatorPlugin.class, + MustachePlugin.class)); @SafeVarargs public PreBuiltTransportClient(Settings settings, Class... plugins) { @@ -50,14 +67,25 @@ public class PreBuiltTransportClient extends TransportClient { super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS)); } - /** - * The default transport implementation for the transport client. - */ - public static final class TransportPlugin extends Netty3Plugin { - // disable assertions for permissions since we might not have the permissions here - // compared to if we are loaded as a real module to the es server - public TransportPlugin(Settings settings) { - super(Settings.builder().put("netty.assert.buglevel", false).put(settings).build()); + public static final class TransportPlugin extends Plugin { + + private static final Setting ASSERT_NETTY_BUGLEVEL = + Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope); + + @Override + public List> getSettings() { + return Collections.singletonList(ASSERT_NETTY_BUGLEVEL); } + + @Override + public Settings additionalSettings() { + return Settings.builder() + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME) + .put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME) + .put("netty.assert.buglevel", true) + .build(); + } + } + } diff --git a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java index ed300b70021..5b72006f5f0 100644 --- a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java +++ b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.transport.client; import com.carrotsearch.randomizedtesting.RandomizedTest; @@ -57,4 +58,5 @@ public class PreBuiltTransportClientTests extends RandomizedTest { } } } + } diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java index 0925c744144..49ac5d4f8a4 100644 --- a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.StatusToXContent; @@ -109,6 +110,30 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr return getShardInfo().status(); } + /** + * Gets the location of the written document as a string suitable for a {@code Location} header. + * @param routing any routing used in the request. If null the location doesn't include routing information. + */ + public String getLocation(@Nullable String routing) { + // Absolute path for the location of the document. This should be allowed as of HTTP/1.1: + // https://tools.ietf.org/html/rfc7231#section-7.1.2 + String index = getIndex(); + String type = getType(); + String id = getId(); + String routingStart = "?routing="; + int bufferSize = 3 + index.length() + type.length() + id.length(); + if (routing != null) { + bufferSize += routingStart.length() + routing.length(); + } + StringBuilder location = new StringBuilder(bufferSize); + location.append('/').append(index); + location.append('/').append(type); + location.append('/').append(id); + if (routing != null) { + location.append(routingStart).append(routing); + } + return location.toString(); + } @Override public void readFrom(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index bdec058b04a..7fa6245d1ec 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -142,7 +142,8 @@ final class Bootstrap { JvmInfo.jvmInfo(); } - private void setup(boolean addShutdownHook, Settings settings, Environment environment) throws Exception { + private void setup(boolean addShutdownHook, Environment environment) throws Exception { + Settings settings = environment.settings(); initializeNatives( environment.tmpFile(), BootstrapSettings.MEMORY_LOCK_SETTING.get(settings), @@ -171,7 +172,7 @@ final class Bootstrap { // install SM after natives, shutdown hooks, etc. Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings)); - node = new Node(settings) { + node = new Node(environment) { @Override protected void validateNodeBeforeAcceptingRequests(Settings settings, BoundTransportAddress boundTransportAddress) { BootstrapCheck.check(settings, boundTransportAddress); @@ -179,7 +180,7 @@ final class Bootstrap { }; } - private static Environment initialSettings(boolean foreground, Path pidFile, Map esSettings) { + private static Environment initialEnvironment(boolean foreground, Path pidFile, Map esSettings) { Terminal terminal = foreground ? Terminal.DEFAULT : null; Settings.Builder builder = Settings.builder(); if (pidFile != null) { @@ -225,9 +226,8 @@ final class Bootstrap { INSTANCE = new Bootstrap(); - Environment environment = initialSettings(foreground, pidFile, esSettings); - Settings settings = environment.settings(); - LogConfigurator.configure(settings, true); + Environment environment = initialEnvironment(foreground, pidFile, esSettings); + LogConfigurator.configure(environment.settings(), true); checkForCustomConfFile(); if (environment.pidFile() != null) { @@ -250,9 +250,9 @@ final class Bootstrap { // initialized as we do not want to grant the runtime permission // setDefaultUncaughtExceptionHandler Thread.setDefaultUncaughtExceptionHandler( - new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(settings))); + new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(environment.settings()))); - INSTANCE.setup(true, settings, environment); + INSTANCE.setup(true, environment); INSTANCE.start(); @@ -266,7 +266,7 @@ final class Bootstrap { } ESLogger logger = Loggers.getLogger(Bootstrap.class); if (INSTANCE.node != null) { - logger = Loggers.getLogger(Bootstrap.class, INSTANCE.node.settings().get("node.name")); + logger = Loggers.getLogger(Bootstrap.class, Node.NODE_NAME_SETTING.get(INSTANCE.node.settings())); } // HACK, it sucks to do this, but we will run users out of disk space otherwise if (e instanceof CreationException) { diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 577c00178b6..a50a1e7bbea 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -19,15 +19,6 @@ package org.elasticsearch.client.transport; -import java.io.Closeable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; @@ -63,6 +54,15 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.TransportService; +import java.io.Closeable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + /** * The transport client allows to create a client that is not part of the cluster, but simply connects to one * or more nodes directly by adding their respective addresses using {@link #addTransportAddress(org.elasticsearch.common.transport.TransportAddress)}. @@ -100,6 +100,9 @@ public abstract class TransportClient extends AbstractClient { private static ClientTemplate buildTemplate(Settings providedSettings, Settings defaultSettings, Collection> plugins) { + if (Node.NODE_NAME_SETTING.exists(providedSettings) == false) { + providedSettings = Settings.builder().put(providedSettings).put(Node.NODE_NAME_SETTING.getKey(), "_client_").build(); + } final PluginsService pluginsService = newPluginService(providedSettings, plugins); final Settings settings = Settings.builder().put(defaultSettings).put(pluginsService.updatedSettings()).build(); final List resourcesToClose = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java index 5f2cb10212d..0c39c43bc9f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java +++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java @@ -39,7 +39,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.function.Predicate; -import java.util.function.Supplier; import static org.elasticsearch.common.transport.TransportAddressSerializers.addressToStream; @@ -191,7 +190,7 @@ public class DiscoveryNode implements Writeable, ToXContent { } /** Creates a DiscoveryNode representing the local node. */ - public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeIdSupplier) { + public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeId) { Map attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(settings).getAsMap()); Set roles = new HashSet<>(); if (Node.NODE_INGEST_SETTING.get(settings)) { @@ -204,8 +203,7 @@ public class DiscoveryNode implements Writeable, ToXContent { roles.add(DiscoveryNode.Role.DATA); } - return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeIdSupplier, publishAddress, - attributes, roles, Version.CURRENT); + return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress,attributes, roles, Version.CURRENT); } /** diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java index 63afe9a0323..955b836ca1c 100644 --- a/core/src/main/java/org/elasticsearch/common/Strings.java +++ b/core/src/main/java/org/elasticsearch/common/Strings.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; @@ -509,7 +510,19 @@ public class Strings { else return s.split(","); } + /** + * A convenience method for splitting a delimited string into + * a set and trimming leading and trailing whitespace from all + * split strings. + * + * @param s the string to split + * @param c the delimiter to split on + * @return the set of split strings + */ public static Set splitStringToSet(final String s, final char c) { + if (s == null || s.isEmpty()) { + return Collections.emptySet(); + } final char[] chars = s.toCharArray(); int count = 1; for (final char x : chars) { @@ -521,16 +534,25 @@ public class Strings { final int len = chars.length; int start = 0; // starting index in chars of the current substring. int pos = 0; // current index in chars. + int end = 0; // the position of the end of the current token for (; pos < len; pos++) { if (chars[pos] == c) { - int size = pos - start; + int size = end - start; if (size > 0) { // only add non empty strings result.add(new String(chars, start, size)); } start = pos + 1; + end = start; + } else if (Character.isWhitespace(chars[pos])) { + if (start == pos) { + // skip over preceding whitespace + start++; + } + } else { + end = pos + 1; } } - int size = pos - start; + int size = end - start; if (size > 0) { result.add(new String(chars, start, size)); } diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index ef1102326d6..b336acfba20 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.bytes; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; @@ -36,24 +35,24 @@ public class PagedBytesReference extends BytesReference { private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE; private final BigArrays bigarrays; - protected final ByteArray bytearray; + protected final ByteArray byteArray; private final int offset; private final int length; - public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) { - this(bigarrays, bytearray, 0, length); + public PagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) { + this(bigarrays, byteArray, 0, length); } - public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int from, int length) { + public PagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int from, int length) { this.bigarrays = bigarrays; - this.bytearray = bytearray; + this.byteArray = byteArray; this.offset = from; this.length = length; } @Override public byte get(int index) { - return bytearray.get(offset + index); + return byteArray.get(offset + index); } @Override @@ -66,14 +65,14 @@ public class PagedBytesReference extends BytesReference { if (from < 0 || (from + length) > length()) { throw new IllegalArgumentException("can't slice a buffer with length [" + length() + "], with slice parameters from [" + from + "], length [" + length + "]"); } - return new PagedBytesReference(bigarrays, bytearray, offset + from, length); + return new PagedBytesReference(bigarrays, byteArray, offset + from, length); } @Override public BytesRef toBytesRef() { BytesRef bref = new BytesRef(); // if length <= pagesize this will dereference the page, or materialize the byte[] - bytearray.get(offset, length, bref); + byteArray.get(offset, length, bref); return bref; } @@ -95,7 +94,7 @@ public class PagedBytesReference extends BytesReference { @Override public BytesRef next() throws IOException { if (nextFragmentSize != 0) { - final boolean materialized = bytearray.get(offset + position, nextFragmentSize, slice); + final boolean materialized = byteArray.get(offset + position, nextFragmentSize, slice); assert materialized == false : "iteration should be page aligned but array got materialized"; position += nextFragmentSize; final int remaining = length - position; @@ -111,6 +110,6 @@ public class PagedBytesReference extends BytesReference { @Override public long ramBytesUsed() { - return bytearray.ramBytesUsed(); + return byteArray.ramBytesUsed(); } } diff --git a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java index 2152aa226a8..2700ea4dc13 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java @@ -30,12 +30,13 @@ import org.elasticsearch.common.util.ByteArray; */ public final class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable { - public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) { - super(bigarrays, bytearray, length); + public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) { + super(bigarrays, byteArray, length); } @Override public void close() { - Releasables.close(bytearray); + Releasables.close(byteArray); } + } diff --git a/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java b/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java index d2fb9e3ecb9..fa49a80123d 100644 --- a/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java +++ b/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; /** * @@ -50,7 +51,7 @@ public abstract class AbstractComponent { * Returns the nodes name from the settings or the empty string if not set. */ public final String nodeName() { - return settings.get("node.name", ""); + return Node.NODE_NAME_SETTING.get(settings); } /** diff --git a/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java b/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java index 15bc324a346..e31f206bcad 100644 --- a/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java +++ b/core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java @@ -28,4 +28,5 @@ public interface ReleasableBytesStream extends BytesStream { @Override ReleasablePagedBytesReference bytes(); + } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java index 2d2719a113e..3de5c757ae1 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java @@ -33,7 +33,7 @@ import java.io.IOException; */ public class BytesStreamOutput extends StreamOutput implements BytesStream { - protected final BigArrays bigarrays; + protected final BigArrays bigArrays; protected ByteArray bytes; protected int count; @@ -57,9 +57,9 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { this(expectedSize, BigArrays.NON_RECYCLING_INSTANCE); } - protected BytesStreamOutput(int expectedSize, BigArrays bigarrays) { - this.bigarrays = bigarrays; - this.bytes = bigarrays.newByteArray(expectedSize); + protected BytesStreamOutput(int expectedSize, BigArrays bigArrays) { + this.bigArrays = bigArrays; + this.bytes = bigArrays.newByteArray(expectedSize); } @Override @@ -100,7 +100,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { public void reset() { // shrink list of pages if (bytes.size() > BigArrays.PAGE_SIZE_IN_BYTES) { - bytes = bigarrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES); + bytes = bigArrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES); } // go back to start @@ -145,7 +145,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { @Override public BytesReference bytes() { - return new PagedBytesReference(bigarrays, bytes, count); + return new PagedBytesReference(bigArrays, bytes, count); } /** @@ -157,7 +157,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream { } private void ensureCapacity(int offset) { - bytes = bigarrays.grow(bytes, offset); + bytes = bigArrays.grow(bytes, offset); } } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java index 4e5380e66d5..674ff18f0fc 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java @@ -36,12 +36,13 @@ public class ReleasableBytesStreamOutput extends BytesStreamOutput implements Re super(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays); } - public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigarrays) { - super(expectedSize, bigarrays); + public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigArrays) { + super(expectedSize, bigArrays); } @Override public ReleasablePagedBytesReference bytes() { - return new ReleasablePagedBytesReference(bigarrays, bytes, count); + return new ReleasablePagedBytesReference(bigArrays, bytes, count); } + } diff --git a/core/src/main/java/org/elasticsearch/common/lease/Releasable.java b/core/src/main/java/org/elasticsearch/common/lease/Releasable.java index a86d812016e..61b1e85c749 100644 --- a/core/src/main/java/org/elasticsearch/common/lease/Releasable.java +++ b/core/src/main/java/org/elasticsearch/common/lease/Releasable.java @@ -30,4 +30,5 @@ public interface Releasable extends Closeable { @Override void close(); + } diff --git a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java index 613193baee5..4a938e38a2e 100644 --- a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java +++ b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.node.Node; import java.net.InetAddress; import java.net.UnknownHostException; @@ -101,9 +102,8 @@ public class Loggers { prefixesList.add(addr.getHostName()); } } - String name = settings.get("node.name"); - if (name != null) { - prefixesList.add(name); + if (Node.NODE_NAME_SETTING.exists(settings)) { + prefixesList.add(Node.NODE_NAME_SETTING.get(settings)); } if (prefixes != null && prefixes.length > 0) { prefixesList.addAll(asList(prefixes)); diff --git a/core/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java b/core/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java index 77a39a8c22b..0317026b6be 100644 --- a/core/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java +++ b/core/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java @@ -55,6 +55,9 @@ public class NetworkExceptionHelper { if (e.getMessage().contains("Connection timed out")) { return true; } + if (e.getMessage().equals("Socket is closed")) { + return true; + } } return false; } diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java index da49aac5ed0..6a15a3d9000 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java @@ -429,6 +429,10 @@ public class BigArrays implements Releasable { return this.circuitBreakingInstance; } + public CircuitBreakerService breakerService() { + return this.circuitBreakingInstance.breakerService; + } + private T resizeInPlace(T array, long newSize) { final long oldMemSize = array.ramBytesUsed(); array.resize(newSize); diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index 5ac94f8b386..2d682648ca4 100644 --- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -22,6 +22,7 @@ package org.elasticsearch.common.util.concurrent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; import java.util.Arrays; import java.util.concurrent.BlockingQueue; @@ -83,11 +84,10 @@ public class EsExecutors { } public static String threadName(Settings settings, String namePrefix) { - String nodeName = settings.get("node.name"); - if (nodeName == null) { - return threadName("", namePrefix); + if (Node.NODE_NAME_SETTING.exists(settings)) { + return threadName(Node.NODE_NAME_SETTING.get(settings), namePrefix); } else { - return threadName(nodeName, namePrefix); + return threadName("", namePrefix); } } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java index d181ffc21d1..f22aa39613f 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java @@ -21,7 +21,8 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.rest.RestStatus; /** - * + * Objects that can both render themselves in as json/yaml/etc and can provide a {@link RestStatus} for their response. Usually should be + * implemented by top level responses sent back to users from REST endpoints. */ public interface StatusToXContent extends ToXContent { diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java index 8c8b72bff63..4b544aa3882 100644 --- a/core/src/main/java/org/elasticsearch/env/Environment.java +++ b/core/src/main/java/org/elasticsearch/env/Environment.java @@ -36,6 +36,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Function; import static org.elasticsearch.common.Strings.cleanPath; @@ -107,7 +108,6 @@ public class Environment { } public Environment(Settings settings) { - this.settings = settings; final Path homeFile; if (PATH_HOME_SETTING.exists(settings)) { homeFile = PathUtils.get(cleanPath(PATH_HOME_SETTING.get(settings))); @@ -171,6 +171,13 @@ public class Environment { binFile = homeFile.resolve("bin"); libFile = homeFile.resolve("lib"); modulesFile = homeFile.resolve("modules"); + + Settings.Builder finalSettings = Settings.builder().put(settings); + finalSettings.put(PATH_HOME_SETTING.getKey(), homeFile); + finalSettings.putArray(PATH_DATA_SETTING.getKey(), dataPaths); + finalSettings.put(PATH_LOGS_SETTING.getKey(), logsFile); + this.settings = finalSettings.build(); + } /** @@ -332,4 +339,26 @@ public class Environment { public static FileStore getFileStore(Path path) throws IOException { return ESFileStore.getMatchingFileStore(path, fileStores); } + + /** + * asserts that the two environments are equivalent for all things the environment cares about (i.e., all but the setting + * object which may contain different setting) + */ + public static void assertEquivalent(Environment actual, Environment expected) { + assertEquals(actual.dataWithClusterFiles(), expected.dataWithClusterFiles(), "dataWithClusterFiles"); + assertEquals(actual.repoFiles(), expected.repoFiles(), "repoFiles"); + assertEquals(actual.configFile(), expected.configFile(), "configFile"); + assertEquals(actual.scriptsFile(), expected.scriptsFile(), "scriptsFile"); + assertEquals(actual.pluginsFile(), expected.pluginsFile(), "pluginsFile"); + assertEquals(actual.binFile(), expected.binFile(), "binFile"); + assertEquals(actual.libFile(), expected.libFile(), "libFile"); + assertEquals(actual.modulesFile(), expected.modulesFile(), "modulesFile"); + assertEquals(actual.logsFile(), expected.logsFile(), "logsFile"); + assertEquals(actual.pidFile(), expected.pidFile(), "pidFile"); + assertEquals(actual.tmpFile(), expected.tmpFile(), "tmpFile"); + } + + private static void assertEquals(Object actual, Object expected, String name) { + assert Objects.deepEquals(actual, expected) : "actual " + name + " [" + actual + "] is different than [ " + expected + "]"; + } } diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 0e4f83b625d..367131d93cd 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -34,10 +34,10 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -52,6 +52,7 @@ import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.fs.FsProbe; import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.node.Node; import java.io.Closeable; import java.io.IOException; @@ -79,7 +80,9 @@ import static java.util.Collections.unmodifiableSet; /** * A component that holds all data paths for a single node. */ -public final class NodeEnvironment extends AbstractComponent implements Closeable { +public final class NodeEnvironment implements Closeable { + + private final ESLogger logger; public static class NodePath { /* ${data.paths}/nodes/{node.id} */ @@ -139,8 +142,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl private final Path sharedDataPath; private final Lock[] locks; - private final boolean addLockIdToCustomPath; - private final int nodeLockId; private final AtomicBoolean closed = new AtomicBoolean(false); private final Map shardLocks = new HashMap<>(); @@ -177,12 +178,8 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl public static final String NODES_FOLDER = "nodes"; public static final String INDICES_FOLDER = "indices"; public static final String NODE_LOCK_FILENAME = "node.lock"; - public static final String UPGRADE_LOCK_FILENAME = "upgrade.lock"; public NodeEnvironment(Settings settings, Environment environment) throws IOException { - super(settings); - - this.addLockIdToCustomPath = ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.get(settings); if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) { nodePaths = null; @@ -190,12 +187,16 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl locks = null; nodeLockId = -1; nodeMetaData = new NodeMetaData(generateNodeId(settings)); + logger = Loggers.getLogger(getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId())); return; } final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length]; final Lock[] locks = new Lock[nodePaths.length]; boolean success = false; + // trace logger to debug issues before the default node name is derived from the node id + ESLogger startupTraceLogger = Loggers.getLogger(getClass(), settings); + try { sharedDataPath = environment.sharedDataFile(); int nodeLockId = -1; @@ -207,7 +208,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl Path dataDir = environment.dataFiles()[dirIndex]; // TODO: Remove this in 6.0, we are no longer going to read from the cluster name directory if (readFromDataPathWithClusterName(dataDirWithClusterName)) { - DeprecationLogger deprecationLogger = new DeprecationLogger(logger); + DeprecationLogger deprecationLogger = new DeprecationLogger(startupTraceLogger); deprecationLogger.deprecated("ES has detected the [path.data] folder using the cluster name as a folder [{}], " + "Elasticsearch 6.0 will not allow the cluster name as a folder within the data path", dataDir); dataDir = dataDirWithClusterName; @@ -216,20 +217,20 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl Files.createDirectories(dir); try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) { - logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath()); + startupTraceLogger.trace("obtaining node lock on {} ...", dir.toAbsolutePath()); try { locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME); nodePaths[dirIndex] = new NodePath(dir); nodeLockId = possibleLockId; } catch (LockObtainFailedException ex) { - logger.trace("failed to obtain node lock on {}", dir.toAbsolutePath()); + startupTraceLogger.trace("failed to obtain node lock on {}", dir.toAbsolutePath()); // release all the ones that were obtained up until now releaseAndNullLocks(locks); break; } } catch (IOException e) { - logger.trace("failed to obtain node lock on {}", e, dir.toAbsolutePath()); + startupTraceLogger.trace("failed to obtain node lock on {}", e, dir.toAbsolutePath()); lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e); // release all the ones that were obtained up until now releaseAndNullLocks(locks); @@ -246,6 +247,8 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl throw new IllegalStateException("Failed to obtain node lock, is the following location writable?: " + Arrays.toString(environment.dataWithClusterFiles()), lastException); } + this.nodeMetaData = loadOrCreateNodeMetaData(settings, startupTraceLogger, nodePaths); + this.logger = Loggers.getLogger(getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId())); this.nodeLockId = nodeLockId; this.locks = locks; @@ -258,8 +261,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl maybeLogPathDetails(); maybeLogHeapDetails(); - this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths); - applySegmentInfosTrace(settings); assertCanWrite(); success = true; @@ -924,10 +925,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl } } - Settings getSettings() { // for testing - return settings; - } - /** * Resolve the custom path for a index's shard. * Uses the {@code IndexMetaData.SETTING_DATA_PATH} setting to determine @@ -940,7 +937,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl if (customDataDir != null) { // This assert is because this should be caught by MetaDataCreateIndexService assert sharedDataPath != null; - if (addLockIdToCustomPath) { + if (ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.get(indexSettings.getNodeSettings())) { return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.nodeLockId)); } else { return sharedDataPath.resolve(customDataDir); diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java b/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java index 7fe5562f17a..16b0bd00443 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java @@ -29,4 +29,5 @@ import org.elasticsearch.rest.RestRequest; public interface HttpServerAdapter { void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext context); + } diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java index ab8b2bd1797..0ec57e2bfc2 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java @@ -22,9 +22,6 @@ package org.elasticsearch.http; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.transport.BoundTransportAddress; -/** - * - */ public interface HttpServerTransport extends LifecycleComponent { BoundTransportAddress boundAddress(); @@ -34,4 +31,5 @@ public interface HttpServerTransport extends LifecycleComponent { HttpStats stats(); void httpServerAdapter(HttpServerAdapter httpServerAdapter); + } diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 72f8f380df8..60bc3449d0b 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -40,9 +40,9 @@ public final class HttpTransportSettings { public static final Setting SETTING_CORS_MAX_AGE = Setting.intSetting("http.cors.max-age", 1728000, Property.NodeScope); public static final Setting SETTING_CORS_ALLOW_METHODS = - new Setting<>("http.cors.allow-methods", "OPTIONS, HEAD, GET, POST, PUT, DELETE", (value) -> value, Property.NodeScope); + new Setting<>("http.cors.allow-methods", "OPTIONS,HEAD,GET,POST,PUT,DELETE", (value) -> value, Property.NodeScope); public static final Setting SETTING_CORS_ALLOW_HEADERS = - new Setting<>("http.cors.allow-headers", "X-Requested-With, Content-Type, Content-Length", (value) -> value, Property.NodeScope); + new Setting<>("http.cors.allow-headers", "X-Requested-With,Content-Type,Content-Length", (value) -> value, Property.NodeScope); public static final Setting SETTING_CORS_ALLOW_CREDENTIALS = Setting.boolSetting("http.cors.allow-credentials", false, Property.NodeScope); public static final Setting SETTING_PIPELINING = diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java index df348a5d6a1..bbbe3b80cd3 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.node.Node; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -227,7 +228,7 @@ public final class IndexSettings { this.index = indexMetaData.getIndex(); version = Version.indexCreated(settings); logger = Loggers.getLogger(getClass(), settings, index); - nodeName = settings.get("node.name", ""); + nodeName = Node.NODE_NAME_SETTING.get(settings); this.indexMetaData = indexMetaData; numberOfShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null); isShadowReplicaIndex = IndexMetaData.isIndexUsingShadowReplicas(settings); diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 04b8e071f5d..3dd7bd0bcc8 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -34,8 +34,8 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsContext; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; @@ -574,12 +574,12 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl } } if (docValueFields != null) { - FieldDataFieldsContext fieldDataFieldsContext = innerHitsContext - .getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY); + DocValueFieldsContext docValueFieldsContext = innerHitsContext + .getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY); for (String field : docValueFields) { - fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(field)); + docValueFieldsContext.add(new DocValueFieldsContext.DocValueField(field)); } - fieldDataFieldsContext.setHitExecutionNeeded(true); + docValueFieldsContext.setHitExecutionNeeded(true); } if (scriptFields != null) { for (ScriptField field : scriptFields) { diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index 328b9564a5d..65571482093 100644 --- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -57,7 +57,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { new Setting<>("indices.breaker.fielddata.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope); public static final Setting REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING = - Setting.byteSizeSetting("indices.breaker.request.limit", "40%", Property.Dynamic, Property.NodeScope); + Setting.byteSizeSetting("indices.breaker.request.limit", "60%", Property.Dynamic, Property.NodeScope); public static final Setting REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING = Setting.doubleSetting("indices.breaker.request.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope); public static final Setting REQUEST_CIRCUIT_BREAKER_TYPE_SETTING = @@ -98,7 +98,10 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.get(settings) ); - this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT, TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(), 1.0, CircuitBreaker.Type.PARENT); + this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT, + TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).bytes(), 1.0, + CircuitBreaker.Type.PARENT); + if (logger.isTraceEnabled()) { logger.trace("parent circuit breaker with settings {}", this.parentSettings); } @@ -137,7 +140,6 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { registerBreaker(newFielddataSettings); HierarchyCircuitBreakerService.this.fielddataSettings = newFielddataSettings; logger.info("Updated breaker settings field data: {}", newFielddataSettings); - } private boolean validateTotalCircuitBreakerLimit(ByteSizeValue byteSizeValue) { @@ -184,7 +186,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { } // Manually add the parent breaker settings since they aren't part of the breaker map allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(), - parentEstimated, 1.0, parentTripCount.get())); + parentEstimated, 1.0, parentTripCount.get())); return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()])); } @@ -207,8 +209,8 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { if (totalUsed > parentLimit) { this.parentTripCount.incrementAndGet(); throw new CircuitBreakingException("[parent] Data too large, data for [" + - label + "] would be larger than limit of [" + - parentLimit + "/" + new ByteSizeValue(parentLimit) + "]", + label + "] would be larger than limit of [" + + parentLimit + "/" + new ByteSizeValue(parentLimit) + "]", totalUsed, parentLimit); } } diff --git a/core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index 4af2e8a811a..501e8c1b2f9 100644 --- a/core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -109,6 +109,7 @@ public class CompoundProcessor implements Processor { throw compoundProcessorException; } else { executeOnFailure(ingestDocument, compoundProcessorException); + break; } } } diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 7966c8d8f49..a57492750ad 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -132,7 +132,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import java.util.stream.Collectors; /** @@ -170,7 +169,16 @@ public class Node implements Closeable { } }, Setting.Property.NodeScope); - + /** + * Adds a default node name to the given setting, if it doesn't already exist + * @return the given setting if node name is already set, or a new copy with a default node name set. + */ + public static final Settings addNodeNameIfNeeded(Settings settings, final String nodeId) { + if (NODE_NAME_SETTING.exists(settings)) { + return settings; + } + return Settings.builder().put(settings).put(NODE_NAME_SETTING.getKey(), nodeId.substring(0, 7)).build(); + } private static final String CLIENT_TYPE = "node"; private final Lifecycle lifecycle = new Lifecycle(); @@ -188,53 +196,82 @@ public class Node implements Closeable { * @param preparedSettings Base settings to configure the node with */ public Node(Settings preparedSettings) { - this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), Collections.>emptyList()); + this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null)); } - protected Node(Environment tmpEnv, Collection> classpathPlugins) { - Settings tmpSettings = Settings.builder().put(tmpEnv.settings()) - .put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build(); + public Node(Environment environment) { + this(environment, Collections.emptyList()); + } + + protected Node(final Environment environment, Collection> classpathPlugins) { final List resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error - - tmpSettings = TribeService.processSettings(tmpSettings); - ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(tmpSettings)); - final String displayVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : ""); - final JvmInfo jvmInfo = JvmInfo.jvmInfo(); - logger.info( - "version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]", - displayVersion, - jvmInfo.pid(), - Build.CURRENT.shortHash(), - Build.CURRENT.date(), - Constants.OS_NAME, - Constants.OS_VERSION, - Constants.OS_ARCH, - Constants.JVM_VENDOR, - Constants.JVM_NAME, - Constants.JAVA_VERSION, - Constants.JVM_VERSION); - - logger.info("initializing ..."); - - if (logger.isDebugEnabled()) { - logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]", - tmpEnv.configFile(), Arrays.toString(tmpEnv.dataFiles()), tmpEnv.logsFile(), tmpEnv.pluginsFile()); - } - // TODO: Remove this in Elasticsearch 6.0.0 - if (JsonXContent.unquotedFieldNamesSet) { - DeprecationLogger dLogger = new DeprecationLogger(logger); - dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0", - JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES); - } - - this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins); - this.settings = pluginsService.updatedSettings(); - // create the environment based on the finalized (processed) view of the settings - this.environment = new Environment(this.settings); - final List> executorBuilders = pluginsService.getExecutorBuilders(settings); - boolean success = false; + { + // use temp logger just to say we are starting. we can't use it later on because the node name might not be set + ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(environment.settings())); + logger.info("initializing ..."); + + } try { + Settings tmpSettings = Settings.builder().put(environment.settings()) + .put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build(); + + tmpSettings = TribeService.processSettings(tmpSettings); + + // create the node environment as soon as possible, to recover the node id and enable logging + try { + nodeEnvironment = new NodeEnvironment(tmpSettings, environment); + resourcesToClose.add(nodeEnvironment); + } catch (IOException ex) { + throw new IllegalStateException("Failed to created node environment", ex); + } + + final boolean hadPredefinedNodeName = NODE_NAME_SETTING.exists(tmpSettings); + tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeEnvironment.nodeId()); + ESLogger logger = Loggers.getLogger(Node.class, tmpSettings); + if (hadPredefinedNodeName == false) { + logger.info("node name [{}] derived from node ID; set [{}] to override", + NODE_NAME_SETTING.get(tmpSettings), NODE_NAME_SETTING.getKey()); + } + + final String displayVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : ""); + final JvmInfo jvmInfo = JvmInfo.jvmInfo(); + logger.info( + "version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]", + displayVersion, + jvmInfo.pid(), + Build.CURRENT.shortHash(), + Build.CURRENT.date(), + Constants.OS_NAME, + Constants.OS_VERSION, + Constants.OS_ARCH, + Constants.JVM_VENDOR, + Constants.JVM_NAME, + Constants.JAVA_VERSION, + Constants.JVM_VERSION); + + + if (logger.isDebugEnabled()) { + logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]", + environment.configFile(), Arrays.toString(environment.dataFiles()), environment.logsFile(), environment.pluginsFile()); + } + // TODO: Remove this in Elasticsearch 6.0.0 + if (JsonXContent.unquotedFieldNamesSet) { + DeprecationLogger dLogger = new DeprecationLogger(logger); + dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0", + JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES); + } + + this.pluginsService = new PluginsService(tmpSettings, environment.modulesFile(), environment.pluginsFile(), classpathPlugins); + this.settings = pluginsService.updatedSettings(); + // create the environment based on the finalized (processed) view of the settings + // this is just to makes sure that people get the same settings, no matter where they ask them from + this.environment = new Environment(this.settings); + Environment.assertEquivalent(environment, this.environment); + + + final List> executorBuilders = pluginsService.getExecutorBuilders(settings); + final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0])); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); // adds the context to the DeprecationLogger so that it does not need to be injected everywhere @@ -249,19 +286,13 @@ public class Node implements Closeable { additionalSettings.addAll(builder.getRegisteredSettings()); } final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); - final ScriptModule scriptModule = ScriptModule.create(settings, environment, resourceWatcherService, - pluginsService.filterPlugins(ScriptPlugin.class)); - AnalysisModule analysisModule = new AnalysisModule(environment, pluginsService.filterPlugins(AnalysisPlugin.class)); + final ScriptModule scriptModule = ScriptModule.create(settings, this.environment, resourceWatcherService, + pluginsService.filterPlugins(ScriptPlugin.class)); + AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); additionalSettings.addAll(scriptModule.getSettings()); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter); - try { - nodeEnvironment = new NodeEnvironment(this.settings, this.environment); - resourcesToClose.add(nodeEnvironment); - } catch (IOException ex) { - throw new IllegalStateException("Failed to created node environment", ex); - } resourcesToClose.add(resourceWatcherService); final NetworkService networkService = new NetworkService(settings); final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool); @@ -270,7 +301,7 @@ public class Node implements Closeable { final TribeService tribeService = new TribeService(settings, clusterService, nodeEnvironment.nodeId()); resourcesToClose.add(tribeService); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); - final IngestService ingestService = new IngestService(settings, threadPool, environment, + final IngestService ingestService = new IngestService(settings, threadPool, this.environment, scriptModule.getScriptService(), pluginsService.filterPlugins(IngestPlugin.class)); ModulesBuilder modules = new ModulesBuilder(); @@ -287,10 +318,10 @@ public class Node implements Closeable { modules.add(new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class))); modules.add(new SearchModule(settings, namedWriteableRegistry, false, pluginsService.filterPlugins(SearchPlugin.class))); modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false, settings, - clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(), - pluginsService.filterPlugins(ActionPlugin.class))); + clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(), + pluginsService.filterPlugins(ActionPlugin.class))); modules.add(new GatewayModule()); - modules.add(new RepositoriesModule(environment, pluginsService.filterPlugins(RepositoryPlugin.class))); + modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class))); pluginsService.processModules(modules); CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(), settingsModule.getClusterSettings()); @@ -306,7 +337,7 @@ public class Node implements Closeable { b.bind(PluginsService.class).toInstance(pluginsService); b.bind(Client.class).toInstance(client); b.bind(NodeClient.class).toInstance(client); - b.bind(Environment.class).toInstance(environment); + b.bind(Environment.class).toInstance(this.environment); b.bind(ThreadPool.class).toInstance(threadPool); b.bind(NodeEnvironment.class).toInstance(nodeEnvironment); b.bind(TribeService.class).toInstance(tribeService); @@ -316,14 +347,14 @@ public class Node implements Closeable { b.bind(ScriptService.class).toInstance(scriptModule.getScriptService()); b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); b.bind(IngestService.class).toInstance(ingestService); - pluginComponents.stream().forEach(p -> b.bind((Class)p.getClass()).toInstance(p)); + pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p)); } ); injector = modules.createInjector(); List pluginLifecycleComponents = pluginComponents.stream() .filter(p -> p instanceof LifecycleComponent) - .map(p -> (LifecycleComponent)p).collect(Collectors.toList()); + .map(p -> (LifecycleComponent) p).collect(Collectors.toList()); pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream() .map(injector::getInstance).collect(Collectors.toList())); resourcesToClose.addAll(pluginLifecycleComponents); @@ -331,6 +362,8 @@ public class Node implements Closeable { client.intialize(injector.getInstance(new Key>() {})); + logger.info("initialized"); + success = true; } catch (IOException ex) { throw new ElasticsearchException("failed to bind service", ex); @@ -339,8 +372,6 @@ public class Node implements Closeable { IOUtils.closeWhileHandlingException(resourcesToClose); } } - - logger.info("initialized"); } /** diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java index 0c0482bfc63..dba7f303130 100644 --- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java +++ b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java @@ -21,18 +21,13 @@ package org.elasticsearch.node.internal; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.env.Environment; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -64,7 +59,7 @@ public class InternalSettingsPreparer { public static Settings prepareSettings(Settings input) { Settings.Builder output = Settings.builder(); initializeSettings(output, input, true, Collections.emptyMap()); - finalizeSettings(output, null, null); + finalizeSettings(output, null); return output.build(); } @@ -120,7 +115,7 @@ public class InternalSettingsPreparer { // re-initialize settings now that the config file has been loaded // TODO: only re-initialize if a config file was actually loaded initializeSettings(output, input, false, properties); - finalizeSettings(output, terminal, environment.configFile()); + finalizeSettings(output, terminal); environment = new Environment(output.build()); @@ -145,9 +140,8 @@ public class InternalSettingsPreparer { /** * Finish preparing settings by replacing forced settings, prompts, and any defaults that need to be added. * The provided terminal is used to prompt for settings needing to be replaced. - * The provided configDir is optional and will be used to lookup names.txt if the node name is not set, if provided. */ - private static void finalizeSettings(Settings.Builder output, Terminal terminal, Path configDir) { + private static void finalizeSettings(Settings.Builder output, Terminal terminal) { // allow to force set properties based on configuration of the settings provided List forcedSettings = new ArrayList<>(); for (String setting : output.internalMap().keySet()) { @@ -167,42 +161,6 @@ public class InternalSettingsPreparer { } replacePromptPlaceholders(output, terminal); - // all settings placeholders have been resolved. resolve the value for the name setting by checking for name, - // then looking for node.name, and finally generate one if needed - String name = output.get("node.name"); - if (name == null || name.isEmpty()) { - name = randomNodeName(configDir); - output.put("node.name", name); - } - } - - private static String randomNodeName(Path configDir) { - InputStream input; - if (configDir != null && Files.exists(configDir.resolve("names.txt"))) { - Path namesPath = configDir.resolve("names.txt"); - try { - input = Files.newInputStream(namesPath); - } catch (IOException e) { - throw new RuntimeException("Failed to load custom names.txt from " + namesPath, e); - } - } else { - input = InternalSettingsPreparer.class.getResourceAsStream("/config/names.txt"); - } - - try { - List names = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) { - String name = reader.readLine(); - while (name != null) { - names.add(name); - name = reader.readLine(); - } - } - int index = Randomness.get().nextInt(names.size()); - return names.get(index); - } catch (IOException e) { - throw new RuntimeException("Could not read node names list", e); - } } private static void replacePromptPlaceholders(Settings.Builder settings, Terminal terminal) { diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index aa00943ed08..f5d4f4eb695 100644 --- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -101,4 +101,5 @@ public abstract class AbstractRestChannel implements RestChannel { public boolean detailedErrorsEnabled() { return detailedErrorsEnabled; } + } diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java index d895285ca8c..2a56313fd8a 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java +++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java @@ -30,6 +30,7 @@ import java.io.IOException; * A channel used to construct bytes / builder based outputs, and send responses. */ public interface RestChannel { + XContentBuilder newBuilder() throws IOException; XContentBuilder newErrorBuilder() throws IOException; @@ -46,4 +47,5 @@ public interface RestChannel { boolean detailedErrorsEnabled(); void sendResponse(RestResponse response); + } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index 7072c4d59de..cf7c811a84c 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -193,7 +193,7 @@ public class RestController extends AbstractLifecycleComponent { if (!checkRequestParameters(request, channel)) { return; } - try (ThreadContext.StoredContext t = threadContext.stashContext()) { + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { for (String key : headersToCopy) { String httpHeader = request.header(key); if (httpHeader != null) { diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java index 8872484d589..897ba294039 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -29,16 +29,35 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.rest.support.RestUtils; import java.net.SocketAddress; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue; import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; -/** - * - */ public abstract class RestRequest implements ToXContent.Params { + private final Map params; + private final String rawPath; + + public RestRequest(String uri) { + final Map params = new HashMap<>(); + int pathEndPos = uri.indexOf('?'); + if (pathEndPos < 0) { + this.rawPath = uri; + } else { + this.rawPath = uri.substring(0, pathEndPos); + RestUtils.decodeQueryString(uri, pathEndPos + 1, params); + } + this.params = params; + } + + public RestRequest(Map params, String path) { + this.params = params; + this.rawPath = path; + } + public enum Method { GET, POST, PUT, DELETE, OPTIONS, HEAD } @@ -53,7 +72,9 @@ public abstract class RestRequest implements ToXContent.Params { /** * The non decoded, raw path provided. */ - public abstract String rawPath(); + public String rawPath() { + return rawPath; + } /** * The path part of the URI (without the query string), decoded. @@ -80,12 +101,27 @@ public abstract class RestRequest implements ToXContent.Params { return null; } - public abstract boolean hasParam(String key); + public final boolean hasParam(String key) { + return params.containsKey(key); + } @Override - public abstract String param(String key); + public final String param(String key) { + return params.get(key); + } - public abstract Map params(); + @Override + public final String param(String key, String defaultValue) { + String value = params.get(key); + if (value == null) { + return defaultValue; + } + return value; + } + + public Map params() { + return params; + } public float paramAsFloat(String key, float defaultValue) { String sValue = param(key); diff --git a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java index c3c4549aa63..a4222adacd7 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java @@ -90,7 +90,8 @@ public class RestIndexAction extends BaseRestHandler { } catch (IllegalArgumentException eia){ try { XContentBuilder builder = channel.newErrorBuilder(); - channel.sendResponse(new BytesRestResponse(BAD_REQUEST, builder.startObject().field("error", eia.getMessage()).endObject())); + channel.sendResponse( + new BytesRestResponse(BAD_REQUEST, builder.startObject().field("error", eia.getMessage()).endObject())); } catch (IOException e1) { logger.warn("Failed to send response", e1); return; @@ -101,6 +102,6 @@ public class RestIndexAction extends BaseRestHandler { if (consistencyLevel != null) { indexRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); } - client.index(indexRequest, new RestStatusToXContentListener<>(channel)); + client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing()))); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestStatusToXContentListener.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestStatusToXContentListener.java index 6bd9a900c8e..3e4677ab446 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/support/RestStatusToXContentListener.java +++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestStatusToXContentListener.java @@ -23,14 +23,30 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; + +import java.util.function.Function; /** - * + * Content listener that extracts that {@link RestStatus} from the response. */ public class RestStatusToXContentListener extends RestResponseListener { + private final Function extractLocation; + /** + * Build an instance that doesn't support responses with the status {@code 201 CREATED}. + */ public RestStatusToXContentListener(RestChannel channel) { + // TODO switch this to throwing an exception? + this(channel, r -> null); + } + + /** + * Build an instance that does support responses with the status {@code 201 CREATED}. + */ + public RestStatusToXContentListener(RestChannel channel, Function extractLocation) { super(channel); + this.extractLocation = extractLocation; } @Override @@ -42,7 +58,13 @@ public class RestStatusToXContentListener ext builder.startObject(); response.toXContent(builder, channel.request()); builder.endObject(); - return new BytesRestResponse(response.status(), builder); + BytesRestResponse restResponse = new BytesRestResponse(response.status(), builder); + if (RestStatus.CREATED == restResponse.status()) { + String location = extractLocation.apply(response); + if (location != null) { + restResponse.addHeader("Location", location); + } + } + return restResponse; } - } diff --git a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java index ddd52001e03..cfe26d35c56 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java @@ -97,6 +97,6 @@ public class RestUpdateAction extends BaseRestHandler { } } - client.update(updateRequest, new RestStatusToXContentListener<>(channel)); + client.update(updateRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(updateRequest.routing()))); } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index 06f1ab1f1ad..c6ce4a040f3 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -246,7 +246,7 @@ import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase; import org.elasticsearch.search.fetch.parent.ParentFieldSubFetchPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase; @@ -792,7 +792,7 @@ public class SearchModule extends AbstractModule { private void registerFetchSubPhases(List plugins) { registerFetchSubPhase(new ExplainFetchSubPhase()); - registerFetchSubPhase(new FieldDataFieldsFetchSubPhase()); + registerFetchSubPhase(new DocValueFieldsFetchSubPhase()); registerFetchSubPhase(new ScriptFieldsFetchSubPhase()); registerFetchSubPhase(new FetchSourceSubPhase()); registerFetchSubPhase(new VersionFetchSubPhase()); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 976e0cb7939..bfcfcb9d4c8 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; -import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -67,9 +66,9 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.QueryFetchSearchResult; import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult; import org.elasticsearch.search.fetch.ShardFetchRequest; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsContext; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsContext.DocValueField; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsContext.ScriptField; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.DefaultSearchContext; @@ -726,11 +725,11 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv context.fetchSourceContext(source.fetchSource()); } if (source.docValueFields() != null) { - FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY); + DocValueFieldsContext docValuesFieldsContext = context.getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY); for (String field : source.docValueFields()) { - fieldDataFieldsContext.add(new FieldDataField(field)); + docValuesFieldsContext.add(new DocValueField(field)); } - fieldDataFieldsContext.setHitExecutionNeeded(true); + docValuesFieldsContext.setHitExecutionNeeded(true); } if (source.highlighter() != null) { HighlightBuilder highlightBuilder = source.highlighter(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java index 04023b04977..c99da85f331 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java @@ -18,7 +18,10 @@ */ package org.elasticsearch.search.aggregations; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.aggregations.bucket.BestBucketsDeferringCollector; import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; @@ -37,6 +40,9 @@ import java.util.Map; */ public abstract class AggregatorBase extends Aggregator { + /** The default "weight" that a bucket takes when performing an aggregation */ + public static final int DEFAULT_WEIGHT = 1024 * 5; // 5kb + protected final String name; protected final Aggregator parent; protected final AggregationContext context; @@ -48,6 +54,8 @@ public abstract class AggregatorBase extends Aggregator { private Map subAggregatorbyName; private DeferringBucketCollector recordingWrapper; private final List pipelineAggregators; + private final CircuitBreakerService breakerService; + private boolean failed = false; /** * Constructs a new Aggregator. @@ -65,6 +73,7 @@ public abstract class AggregatorBase extends Aggregator { this.metaData = metaData; this.parent = parent; this.context = context; + this.breakerService = context.bigArrays().breakerService(); assert factories != null : "sub-factories provided to BucketAggregator must not be null, use AggragatorFactories.EMPTY instead"; this.subAggregators = factories.createSubAggregators(this); context.searchContext().addReleasable(this, Lifetime.PHASE); @@ -96,6 +105,14 @@ public abstract class AggregatorBase extends Aggregator { return false; // unreachable } }; + try { + this.breakerService + .getBreaker(CircuitBreaker.REQUEST) + .addEstimateBytesAndMaybeBreak(DEFAULT_WEIGHT, ""); + } catch (CircuitBreakingException cbe) { + this.failed = true; + throw cbe; + } } /** @@ -245,7 +262,13 @@ public abstract class AggregatorBase extends Aggregator { /** Called upon release of the aggregator. */ @Override public void close() { - doClose(); + try { + doClose(); + } finally { + if (!this.failed) { + this.breakerService.getBreaker(CircuitBreaker.REQUEST).addWithoutBreaking(-DEFAULT_WEIGHT); + } + } } /** Release instance-specific data. */ diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java index 854838b7441..44ecacd8417 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java @@ -229,6 +229,10 @@ public abstract class AggregatorFactory> { return type.name(); } + public AggregatorFactory getParent() { + return parent; + } + /** * Utility method. Given an {@link AggregatorFactory} that creates * {@link Aggregator}s that only know how to collect bucket 0, this @@ -241,4 +245,4 @@ public abstract class AggregatorFactory> { return new MultiBucketAggregatorWrapper(bigArrays, context, parent, factory, first); } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java index f1e8b7358c0..a1326aaed11 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java @@ -43,7 +43,7 @@ public abstract class InternalMultiBucketAggregation path) { if (path.isEmpty()) { return this; + } else if (path.get(0).equals("_bucket_count")) { + return getBuckets().size(); } else { List buckets = getBuckets(); Object[] propertyArray = new Object[buckets.size()]; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java index dc47cc09578..98c5f02af05 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java @@ -25,8 +25,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.InternalAggregation.Type; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -80,7 +82,22 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder doBuild(AggregationContext context, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { - return new NestedAggregatorFactory(name, type, path, context, parent, subFactoriesBuilder, metaData); + ObjectMapper childObjectMapper = context.searchContext().getObjectMapper(path); + if (childObjectMapper == null) { + // in case the path has been unmapped: + return new NestedAggregatorFactory(name, type, null, null, context, parent, subFactoriesBuilder, metaData); + } + + if (childObjectMapper.nested().isNested() == false) { + throw new AggregationExecutionException("[nested] nested path [" + path + "] is not nested"); + } + try { + ObjectMapper parentObjectMapper = context.searchContext().getQueryShardContext().nestedScope().nextLevel(childObjectMapper); + return new NestedAggregatorFactory(name, type, parentObjectMapper, childObjectMapper, context, parent, subFactoriesBuilder, + metaData); + } finally { + context.searchContext().getQueryShardContext().nestedScope().previousLevel(); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java index 677951c52c0..7716810a1b3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -44,69 +44,40 @@ import java.io.IOException; import java.util.List; import java.util.Map; -/** - * - */ public class NestedAggregator extends SingleBucketAggregator { static final ParseField PATH_FIELD = new ParseField("path"); - private BitSetProducer parentFilter; + private final BitSetProducer parentFilter; private final Query childFilter; - private DocIdSetIterator childDocs; - private BitSet parentDocs; - - public NestedAggregator(String name, AggregatorFactories factories, ObjectMapper objectMapper, AggregationContext aggregationContext, Aggregator parentAggregator, List pipelineAggregators, Map metaData) throws IOException { + public NestedAggregator(String name, AggregatorFactories factories, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, + AggregationContext aggregationContext, Aggregator parentAggregator, + List pipelineAggregators, Map metaData) throws IOException { super(name, factories, aggregationContext, parentAggregator, pipelineAggregators, metaData); - childFilter = objectMapper.nestedTypeFilter(); + Query parentFilter = parentObjectMapper != null ? parentObjectMapper.nestedTypeFilter() : Queries.newNonNestedFilter(); + this.parentFilter = context.searchContext().bitsetFilterCache().getBitSetProducer(parentFilter); + this.childFilter = childObjectMapper.nestedTypeFilter(); } @Override public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { - // Reset parentFilter, so we resolve the parentDocs for each new segment being searched - this.parentFilter = null; - final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx); - final IndexSearcher searcher = new IndexSearcher(topLevelContext); + IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx); + IndexSearcher searcher = new IndexSearcher(topLevelContext); searcher.setQueryCache(null); - final Weight weight = searcher.createNormalizedWeight(childFilter, false); + Weight weight = searcher.createNormalizedWeight(childFilter, false); Scorer childDocsScorer = weight.scorer(ctx); - if (childDocsScorer == null) { - childDocs = null; - } else { - childDocs = childDocsScorer.iterator(); - } + final BitSet parentDocs = parentFilter.getBitSet(ctx); + final DocIdSetIterator childDocs = childDocsScorer != null ? childDocsScorer.iterator() : null; return new LeafBucketCollectorBase(sub, null) { @Override public void collect(int parentDoc, long bucket) throws IOException { - // here we translate the parent doc to a list of its nested docs, and then call super.collect for evey one of them so they'll be collected - - // if parentDoc is 0 then this means that this parent doesn't have child docs (b/c these appear always before the parent doc), so we can skip: - if (parentDoc == 0 || childDocs == null) { + // if parentDoc is 0 then this means that this parent doesn't have child docs (b/c these appear always before the parent + // doc), so we can skip: + if (parentDoc == 0 || parentDocs == null || childDocs == null) { return; } - if (parentFilter == null) { - // The aggs are instantiated in reverse, first the most inner nested aggs and lastly the top level aggs - // So at the time a nested 'nested' aggs is parsed its closest parent nested aggs hasn't been constructed. - // So the trick is to set at the last moment just before needed and we can use its child filter as the - // parent filter. - - // Additional NOTE: Before this logic was performed in the setNextReader(...) method, but the assumption - // that aggs instances are constructed in reverse doesn't hold when buckets are constructed lazily during - // aggs execution - Query parentFilterNotCached = findClosestNestedPath(parent()); - if (parentFilterNotCached == null) { - parentFilterNotCached = Queries.newNonNestedFilter(); - } - parentFilter = context.searchContext().bitsetFilterCache().getBitSetProducer(parentFilterNotCached); - parentDocs = parentFilter.getBitSet(ctx); - if (parentDocs == null) { - // There are no parentDocs in the segment, so return and set childDocs to null, so we exit early for future invocations. - childDocs = null; - return; - } - } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); int childDocId = childDocs.docID(); @@ -123,8 +94,8 @@ public class NestedAggregator extends SingleBucketAggregator { @Override public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException { - return new InternalNested(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal), pipelineAggregators(), - metaData()); + return new InternalNested(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal), + pipelineAggregators(), metaData()); } @Override @@ -132,15 +103,4 @@ public class NestedAggregator extends SingleBucketAggregator { return new InternalNested(name, 0, buildEmptySubAggregations(), pipelineAggregators(), metaData()); } - private static Query findClosestNestedPath(Aggregator parent) { - for (; parent != null; parent = parent.parent()) { - if (parent instanceof NestedAggregator) { - return ((NestedAggregator) parent).childFilter; - } else if (parent instanceof ReverseNestedAggregator) { - return ((ReverseNestedAggregator) parent).getParentFilter(); - } - } - return null; - } - } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java index b09e3d4d40b..ef19a186d52 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java @@ -36,12 +36,15 @@ import java.util.Map; public class NestedAggregatorFactory extends AggregatorFactory { - private final String path; + private final ObjectMapper parentObjectMapper; + private final ObjectMapper childObjectMapper; - public NestedAggregatorFactory(String name, Type type, String path, AggregationContext context, AggregatorFactory parent, - AggregatorFactories.Builder subFactories, Map metaData) throws IOException { + public NestedAggregatorFactory(String name, Type type, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, + AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, + Map metaData) throws IOException { super(name, type, context, parent, subFactories, metaData); - this.path = path; + this.parentObjectMapper = parentObjectMapper; + this.childObjectMapper = childObjectMapper; } @Override @@ -50,14 +53,10 @@ public class NestedAggregatorFactory extends AggregatorFactory { @@ -82,7 +88,40 @@ public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder< @Override protected AggregatorFactory doBuild(AggregationContext context, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { - return new ReverseNestedAggregatorFactory(name, type, path, context, parent, subFactoriesBuilder, metaData); + if (findNestedAggregatorFactory(parent) == null) { + throw new SearchParseException(context.searchContext(), + "Reverse nested aggregation [" + name + "] can only be used inside a [nested] aggregation", null); + } + + ObjectMapper parentObjectMapper = null; + if (path != null) { + parentObjectMapper = context.searchContext().getObjectMapper(path); + if (parentObjectMapper == null) { + return new ReverseNestedAggregatorFactory(name, type, true, null, context, parent, subFactoriesBuilder, metaData); + } + if (parentObjectMapper.nested().isNested() == false) { + throw new AggregationExecutionException("[reverse_nested] nested path [" + path + "] is not nested"); + } + } + + NestedScope nestedScope = context.searchContext().getQueryShardContext().nestedScope(); + try { + nestedScope.nextLevel(parentObjectMapper); + return new ReverseNestedAggregatorFactory(name, type, false, parentObjectMapper, context, parent, subFactoriesBuilder, + metaData); + } finally { + nestedScope.previousLevel(); + } + } + + private static NestedAggregatorFactory findNestedAggregatorFactory(AggregatorFactory parent) { + if (parent == null) { + return null; + } else if (parent instanceof NestedAggregatorFactory) { + return (NestedAggregatorFactory) parent; + } else { + return findNestedAggregatorFactory(parent.getParent()); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java index b8be6479436..d626c1b5441 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java @@ -37,46 +37,26 @@ import java.util.Map; public class ReverseNestedAggregatorFactory extends AggregatorFactory { - private final String path; + private final boolean unmapped; + private final ObjectMapper parentObjectMapper; - public ReverseNestedAggregatorFactory(String name, Type type, String path, AggregationContext context, AggregatorFactory parent, - AggregatorFactories.Builder subFactories, Map metaData) throws IOException { + public ReverseNestedAggregatorFactory(String name, Type type, boolean unmapped, ObjectMapper parentObjectMapper, + AggregationContext context, AggregatorFactory parent, + AggregatorFactories.Builder subFactories, + Map metaData) throws IOException { super(name, type, context, parent, subFactories, metaData); - this.path = path; + this.unmapped = unmapped; + this.parentObjectMapper = parentObjectMapper; } @Override public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - // Early validation - NestedAggregator closestNestedAggregator = findClosestNestedAggregator(parent); - if (closestNestedAggregator == null) { - throw new SearchParseException(context.searchContext(), - "Reverse nested aggregation [" + name + "] can only be used inside a [nested] aggregation", null); - } - - final ObjectMapper objectMapper; - if (path != null) { - objectMapper = context.searchContext().getObjectMapper(path); - if (objectMapper == null) { - return new Unmapped(name, context, parent, pipelineAggregators, metaData); - } - if (!objectMapper.nested().isNested()) { - throw new AggregationExecutionException("[reverse_nested] nested path [" + path + "] is not nested"); - } + if (unmapped) { + return new Unmapped(name, context, parent, pipelineAggregators, metaData); } else { - objectMapper = null; + return new ReverseNestedAggregator(name, factories, parentObjectMapper, context, parent, pipelineAggregators, metaData); } - return new ReverseNestedAggregator(name, factories, objectMapper, context, parent, pipelineAggregators, metaData); - } - - private static NestedAggregator findClosestNestedAggregator(Aggregator parent) { - for (; parent != null; parent = parent.parent()) { - if (parent instanceof NestedAggregator) { - return (NestedAggregator) parent; - } - } - return null; } private static final class Unmapped extends NonCollectingAggregator { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index 65a8c24eb08..a3614889bdf 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -28,9 +28,9 @@ import org.elasticsearch.search.aggregations.InternalAggregation.Type; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsContext; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsContext.DocValueField; +import org.elasticsearch.search.fetch.docvalues.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SubSearchContext; @@ -54,12 +54,12 @@ public class TopHitsAggregatorFactory extends AggregatorFactory> sorts; private final HighlightBuilder highlightBuilder; private final List fieldNames; - private final List fieldDataFields; + private final List docValueFields; private final Set scriptFields; private final FetchSourceContext fetchSourceContext; public TopHitsAggregatorFactory(String name, Type type, int from, int size, boolean explain, boolean version, boolean trackScores, - List> sorts, HighlightBuilder highlightBuilder, List fieldNames, List fieldDataFields, + List> sorts, HighlightBuilder highlightBuilder, List fieldNames, List docValueFields, Set scriptFields, FetchSourceContext fetchSourceContext, AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, type, context, parent, subFactories, metaData); @@ -71,7 +71,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory fields = new ArrayList<>(); + private List fields = new ArrayList<>(); - public FieldDataFieldsContext() { + public DocValueFieldsContext() { } - public void add(FieldDataField field) { + public void add(DocValueField field) { this.fields.add(field); } - public List fields() { + public List fields() { return this.fields; } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/docvalues/DocValueFieldsFetchSubPhase.java similarity index 76% rename from core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java rename to core/src/main/java/org/elasticsearch/search/fetch/docvalues/DocValueFieldsFetchSubPhase.java index 81907d0b906..9946920bc62 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/docvalues/DocValueFieldsFetchSubPhase.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.fetch.fielddata; +package org.elasticsearch.search.fetch.docvalues; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -30,24 +30,23 @@ import java.util.ArrayList; import java.util.HashMap; /** - * Query sub phase which pulls data from field data (using the cache if - * available, building it if not). + * Query sub phase which pulls data from doc values * - * Specifying {@code "fielddata_fields": ["field1", "field2"]} + * Specifying {@code "docvalue_fields": ["field1", "field2"]} */ -public final class FieldDataFieldsFetchSubPhase implements FetchSubPhase { +public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { - public static final String[] NAMES = {"fielddata_fields", "fielddataFields"}; - public static final ContextFactory CONTEXT_FACTORY = new ContextFactory() { + public static final String NAME = "docvalue_fields"; + public static final ContextFactory CONTEXT_FACTORY = new ContextFactory() { @Override public String getName() { - return NAMES[0]; + return NAME; } @Override - public FieldDataFieldsContext newContextInstance() { - return new FieldDataFieldsContext(); + public DocValueFieldsContext newContextInstance() { + return new DocValueFieldsContext(); } }; @@ -56,7 +55,7 @@ public final class FieldDataFieldsFetchSubPhase implements FetchSubPhase { if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) { return; } - for (FieldDataFieldsContext.FieldDataField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) { + for (DocValueFieldsContext.DocValueField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) { if (hitContext.hit().fieldsOrNull() == null) { hitContext.hit().fields(new HashMap<>(2)); } diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java index b0139902a42..7145777aad7 100644 --- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -93,6 +93,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -918,19 +919,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions); } }; - try { - sendMessage(targetChannel, message, onRequestSent, false); - } catch (IOException ex) { - if (nodeConnected(node)) { - throw ex; - } else { - // we might got disconnected in between the nodeChannel(node, options) call and the sending - - // in that case throw a subclass of ConnectTransportException since some code retries based on this - // see TransportMasterNodeAction for instance - throw new NodeNotConnectedException(node, "Node not connected"); - } - } - addedReleaseListener = true; + addedReleaseListener = internalSendMessage(targetChannel, message, onRequestSent); } finally { IOUtils.close(stream); if (!addedReleaseListener) { @@ -939,6 +928,25 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i } } + /** + * sends a message view the given channel, using the given callbacks. + * + * @return true if the message was successfully sent or false when an error occurred and the error hanlding logic was activated + * + */ + private boolean internalSendMessage(Channel targetChannel, BytesReference message, Runnable onRequestSent) throws IOException { + boolean success; + try { + sendMessage(targetChannel, message, onRequestSent, false); + success = true; + } catch (IOException ex) { + // passing exception handling to deal with this and raise disconnect events and decide the right logging level + onException(targetChannel, ex); + success = false; + } + return success; + } + /** * Sends back an error response to the caller via the given channel * @param nodeVersion the caller node version @@ -997,9 +1005,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions); } }; - sendMessage(channel, reference, onRequestSent, false); - addedReleaseListener = true; - + addedReleaseListener = internalSendMessage(channel, reference, onRequestSent); } finally { IOUtils.close(stream); if (!addedReleaseListener) { @@ -1071,23 +1077,23 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i if (buffer.get(offset) != 'E' || buffer.get(offset + 1) != 'S') { // special handling for what is probably HTTP if (bufferStartsWith(buffer, offset, "GET ") || - bufferStartsWith(buffer, offset, "POST ") || - bufferStartsWith(buffer, offset, "PUT ") || - bufferStartsWith(buffer, offset, "HEAD ") || - bufferStartsWith(buffer, offset, "DELETE ") || - bufferStartsWith(buffer, offset, "OPTIONS ") || - bufferStartsWith(buffer, offset, "PATCH ") || - bufferStartsWith(buffer, offset, "TRACE ")) { + bufferStartsWith(buffer, offset, "POST ") || + bufferStartsWith(buffer, offset, "PUT ") || + bufferStartsWith(buffer, offset, "HEAD ") || + bufferStartsWith(buffer, offset, "DELETE ") || + bufferStartsWith(buffer, offset, "OPTIONS ") || + bufferStartsWith(buffer, offset, "PATCH ") || + bufferStartsWith(buffer, offset, "TRACE ")) { throw new HttpOnTransportException("This is not a HTTP port"); } // we have 6 readable bytes, show 4 (should be enough) throw new StreamCorruptedException("invalid internal transport message format, got (" - + Integer.toHexString(buffer.get(offset) & 0xFF) + "," - + Integer.toHexString(buffer.get(offset + 1) & 0xFF) + "," - + Integer.toHexString(buffer.get(offset + 2) & 0xFF) + "," - + Integer.toHexString(buffer.get(offset + 3) & 0xFF) + ")"); + + Integer.toHexString(buffer.get(offset) & 0xFF) + "," + + Integer.toHexString(buffer.get(offset + 1) & 0xFF) + "," + + Integer.toHexString(buffer.get(offset + 2) & 0xFF) + "," + + Integer.toHexString(buffer.get(offset + 3) & 0xFF) + ")"); } final int dataLen; @@ -1106,7 +1112,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i // safety against too large frames being sent if (dataLen > NINETY_PER_HEAP_SIZE) { throw new IllegalArgumentException("transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded [" - + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]"); + + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]"); } if (buffer.length() < dataLen + sizeHeaderLength) { @@ -1154,7 +1160,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent i public final void messageReceived(BytesReference reference, Channel channel, String profileName, InetSocketAddress remoteAddress, int messageLengthBytes) throws IOException { final int totalMessageSize = messageLengthBytes + TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE; - transportServiceAdapter.received(totalMessageSize); + transportServiceAdapter.addBytesReceived(totalMessageSize); // we have additional bytes to read, outside of the header boolean hasMessageBytesToRead = (totalMessageSize - TcpHeader.HEADER_SIZE) > 0; StreamInput streamIn = reference.streamInput(); diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java index 16c9ede851b..1fceb5aa1a3 100644 --- a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java +++ b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java @@ -23,9 +23,6 @@ import org.elasticsearch.Version; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; -/** - * - */ public final class TcpTransportChannel implements TransportChannel { private final TcpTransport transport; protected final Version version; diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java index 765bb988777..15164a5d201 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java @@ -625,12 +625,12 @@ public class TransportService extends AbstractLifecycleComponent { final MeanMetric txMetric = new MeanMetric(); @Override - public void received(long size) { + public void addBytesReceived(long size) { rxMetric.inc(size); } @Override - public void sent(long size) { + public void addBytesSent(long size) { txMetric.inc(size); } diff --git a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java index 5a1b9e9b349..34910532a02 100644 --- a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java +++ b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java @@ -21,14 +21,11 @@ package org.elasticsearch.transport; import org.elasticsearch.cluster.node.DiscoveryNode; -/** - * - */ public interface TransportServiceAdapter { - void received(long size); + void addBytesReceived(long size); - void sent(long size); + void addBytesSent(long size); /** called by the {@link Transport} implementation once a request has been sent */ void onRequestSent(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options); @@ -57,4 +54,5 @@ public interface TransportServiceAdapter { void raiseNodeConnected(DiscoveryNode node); void raiseNodeDisconnected(DiscoveryNode node); + } diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java index eba5fd57734..61559442ff3 100644 --- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java +++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.threadpool.ThreadPool; @@ -228,14 +229,32 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp } final byte[] data = BytesReference.toBytes(stream.bytes()); - transportServiceAdapter.sent(data.length); + transportServiceAdapter.addBytesSent(data.length); transportServiceAdapter.onRequestSent(node, requestId, action, request, options); - targetTransport.workers().execute(() -> { - ThreadContext threadContext = targetTransport.threadPool.getThreadContext(); + targetTransport.receiveMessage(version, data, action, requestId, this); + } + } + + /** + * entry point for incoming messages + * + * @param version the version used to serialize the message + * @param data message data + * @param action the action associated with this message (only used for error handling when data is not parsable) + * @param requestId requestId if the message is request (only used for error handling when data is not parsable) + * @param sourceTransport the source transport to respond to. + */ + public void receiveMessage(Version version, byte[] data, String action, @Nullable Long requestId, LocalTransport sourceTransport) { + try { + workers().execute(() -> { + ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext context = threadContext.stashContext()) { - targetTransport.messageReceived(data, action, LocalTransport.this, version, requestId); + processReceivedMessage(data, action, sourceTransport, version, requestId); } }); + } catch (EsRejectedExecutionException e) { + assert lifecycle.started() == false; + logger.trace("received request but shutting down. ignoring. action [{}], request id [{}]", action, requestId); } } @@ -248,11 +267,12 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS); } - protected void messageReceived(byte[] data, String action, LocalTransport sourceTransport, Version version, - @Nullable final Long sendRequestId) { + /** processes received messages, assuming thread passing and thread context have all been dealt with */ + protected void processReceivedMessage(byte[] data, String action, LocalTransport sourceTransport, Version version, + @Nullable final Long sendRequestId) { Transports.assertTransportThread(); try { - transportServiceAdapter.received(data.length); + transportServiceAdapter.addBytesReceived(data.length); StreamInput stream = StreamInput.wrap(data); stream.setVersion(version); diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java index fc748b96aea..0c1e8747a12 100644 --- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java +++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java @@ -107,12 +107,7 @@ public class LocalTransportChannel implements TransportChannel { private void sendResponseData(byte[] data) { close(); - targetTransport.workers().execute(() -> { - ThreadContext threadContext = targetTransport.threadPool.getThreadContext(); - try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { - targetTransport.messageReceived(data, action, sourceTransport, version, null); - } - }); + targetTransport.receiveMessage(version, data, action, null, sourceTransport); } private void close() { diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java index 572f5e0ce13..eb44a897386 100644 --- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -216,7 +216,7 @@ public class TribeService extends AbstractLifecycleComponent { } } Settings.Builder sb = Settings.builder().put(tribeSettings); - sb.put("node.name", globalSettings.get("node.name") + "/" + tribeName); + sb.put(Node.NODE_NAME_SETTING.getKey(), Node.NODE_NAME_SETTING.get(globalSettings) + "/" + tribeName); sb.put(Environment.PATH_HOME_SETTING.getKey(), Environment.PATH_HOME_SETTING.get(globalSettings)); // pass through ES home dir if (Environment.PATH_CONF_SETTING.exists(globalSettings)) { sb.put(Environment.PATH_CONF_SETTING.getKey(), Environment.PATH_CONF_SETTING.get(globalSettings)); diff --git a/core/src/main/resources/config/names.txt b/core/src/main/resources/config/names.txt deleted file mode 100644 index 9c42c2b10f8..00000000000 --- a/core/src/main/resources/config/names.txt +++ /dev/null @@ -1,2825 +0,0 @@ -3-D Man -A'lars -Aardwolf -Abdul Alhazred -Abe Brown -Abigail Brand -Abner Jenkins -Abner Little -Abominable Snowman -Abomination -Abominatrix -Abraham Cornelius -Abraxas -Absalom -Absorbing Man -Abyss -Access -Achebe -Achelous -Achilles -Acrobat -Adam II -Adam Warlock -Adam X -Adaptoid -Administrator -Adonis -Adrenazon -Adri Nital -Adrian Corbo -Adrian Toomes -Adrienne Frost -Adversary -Advisor -Aegis -Aelfyre Whitemane -Aero -Aftershock -Agamemnon -Agamotto -Agatha Harkness -Aged Genghis -Agent -Agent Axis -Agent Cheesecake -Agent X -Agent Zero -Aggamon -Aginar -Agon -Agony -Agron -Aguja -Ahab -Ahmet Abdol -Ahura -Air-Walker -Airborne -Aireo -Airstrike -Ajak -Ajax -Ajaxis -Akasha -Akhenaten -Al MacKenzie -Alaris -Albert -Albino -Albion -Alchemy -Alcmena -Aldebron -Aleksander Lukin -Aleksei Sytsevich -Aleta Ogord -Alex -Alex Hayden -Alex Power -Alex Wilder -Alexander Bont -Alexander Goodwin Pierce -Alexander Lexington -Alexander Summers -Alfie O'Meggan -Algrim the Strong -Alibar -Alicia Masters -Alistair Smythe -Alistaire Stuart -Aliyah Bishop -Alkhema -All-American -Allatou -Allison Blaire -Alpha Ray -Alpha the Ultimate Mutant -Alyosha Kravinoff -Alysande Stuart -Alyssa Moy -Amahl Farouk -Amalgam -Amanda Sefton -Amatsu-Mikaboshi -Amazon -Amber Hunt -Amelia Voght -Amergin -American Ace -American Dream -American Eagle -American Samurai -Americop -Ameridroid -Amiko Kobayashi -Amina Synge -Aminedi -Ammo -Amphibian -Amphibion -Amphibius -Amun -Anaconda -Anais -Analyzer -Anarchist -Ancient One -Andreas von Strucker -Andrew Chord -Andrew Gervais -Android Man -Andromeda -Anelle -Angar the Screamer -Angel -Angel Dust -Angel Face -Angel Salvadore -Angela Cairn -Angela Del Toro -Angelica Jones -Angelo Unuscione -Angler -Ani-Mator -Animus -Ankhi -Annalee -Anne-Marie Cortez -Annex -Annie Ghazikhanian -Annihilus -Anole -Anomalito -Anomaloco -Anomaly -Answer -Ant-Man -Anthropomorpho -Anti-Cap -Anti-Phoenix Force -Anti-Venom -Anti-Vision -Antimatter -Antiphon the Overseer -Antonio -Anubis -Anvil -Anything -Apache Kid -Apalla -Ape -Ape-Man -Ape-X -Apocalypse -Apollo -Apryll -Aquarian -Aquarius -Aqueduct -Arabian Knight -Arachne -Aragorn -Araki -Aralune -Araña -Arc -Arcade -Arcademan -Arcanna -Archangel -Archenemy -Archer -Archie Corrigan -Archimage -Architect -Arclight -Arcturus Rann -Ardina -Ardroman -Arena -Ares -Argo -Argus -Ariann -Arides -Ariel -Aries -Arishem the Judge -Arize -Arizona Annie -Arkady Rossovich -Arkon -Arkus -Arlette Truffaut -Arlok -Armadillo -Armageddon -Armand Martel -Armor -Armory -Arnim Zola -Arno Stark -Arranger -Arsenal -Arsenic -Artemis -Arthur Parks -Artie -Artie Maddicks -Arturo Falcones -Asbestos Lady -Asbestos Man -Ashcan -Asmodeus -Asp -Assassin -Asteroth -Astra -Astrid Bloom -Astron -Astronomer -Asylum -Atalanta -Atalon -Athena -Atlas -Atleza -Atom Bob -Atom-Smasher -Att-Lass -Attuma -Atum -Aunt May Parker -Auntie Freeze -Auric -Aurora -Authority -Autolycus -Avalanche -Avarrish -Awesome Android -Axum -Azazel -Baal -Balder -Balor -Balthakk -Bandit -Banshee -Bantam -Baphomet -Barbarus -Barnacle -Baron Blood -Baron Brimstone -Baron Macabre -Baron Mordo -Baron Samedi -Baron Strucker -Baron Von Blitzschlag -Baron Zemo -Baroness Blood -Barracuda -Bart Hamilton -Base -Basil Sandhurst -Basilisk -Bast -Bastion -Batragon -Batroc the Leaper -Battering Ram -Battleaxe -Battlestar -Battletide -Batwing -Beast -Beautiful Dreamer -Bedlam -Bedlam II -Beetle -Beetle II -Behemoth -Bela -Belasco -Belathauzer -Bella Donna -Ben Parker -Ben Reilly -Ben Urich -Benazir Kaur -Benedict Kine -Bengal -Benjamin Jacob Grimm -Bennet du Paris -Benny Beckley -Bentley Wittman -Bereet -Berzerker -Bes -Beta Ray Bill -Bethany Cabe -Betty Brant -Betty Brant Leeds -Betty Ross Banner -Bevatron -Beyonder -Bi-Beast -Bible John -Big Bertha -Big Man -Big Wheel -Bill Foster -Binary -Bird-Brain -Bird-Man -Bishop -Bison -Bizarnage -Black Bolt -Black Box -Black Cat -Black Crow -Black Death -Black Dragon -Black Fox -Black Goliath -Black Jack Tarr -Black King -Black Knight -Black Lama -Black Mamba -Black Marvel -Black Panther -Black Queen -Black Talon -Black Tarantula -Black Tom Cassidy -Black Widow -Blackbird -Blackheart -Blackheath -Blacklash -Blackout -Blackwing -Blackwulf -Blade -Blaquesmith -Blastaar -Blaze -Blazing Skull -Blind Faith -Blind Justice -Blindside -Blindspot -Bling -Blink -Blistik -Blitziana -Blitzkrieger -Blizzard -Blizzard II -Blob -Blockbuster -Bloke -Blonde Phantom -Blood Brothers -Blood Rose -Blood Spider -Bloodaxe -Bloodhawk -Bloodlust -Bloodlust II -Bloodscream -Bloodshed -Bloodsport -Bloodstorm -Bloodtide -Bloodwraith -Blowhard -Blue Bullet -Blue Diamond -Blue Marvel -Blue Shield -Blue Streak -Blur -Bob -Bob Diamond -Bobster -Bogeyman -Bombshell -Boneyard -Bonita Juarez -Boobytrap -Book -Boom Boom -Boom Boy -Boomer -Boomerang -Boomslang -Boost -Bora -Bounty -Bounty Hunter -Bova -Box -Box IV -Brain Cell -Brain Drain -Brain-Child -Brainchild -Bram Velsing -Brass -Bres -Brian Braddock -Brian Falsworth -Brigade -Briquette -Brother Nature -Brother Tode -Brother Voodoo -Brothers Grimm -Bruiser -Brunnhilda -Brutacus -Brute I -Brute II -Brute III -Brynocki -Bucky -Bucky III -Bug -Bulldozer -Bullet -Bullseye -Burner -Burstarr -Bushman -Bushmaster -Bushwacker -Butterball -Buzz -Buzzard -Byrrah -Caber -Cable -Cadaver -Cagliostro -Caiera -Caiman -Cain -Cain Marko -Caleb Alexander -Caliban -Callisto -Calvin Rankin -Calypso -Cameron Hodge -Canasta -Cancer -Candra -Cannonball -Cannonball I -Cap 'N Hawk -Caprice -Capricorn -Captain America -Captain Atlas -Captain Barracuda -Captain Britain -Captain Fate -Captain Germany -Captain Marvel -Captain Omen -Captain Savage -Captain UK -Captain Ultra -Captain Universe -Captain Wings -Captain Zero -Cardiac -Cardinal -Caregiver -Caretaker -Carl "Crusher" Creel -Carlos Lobo -Carmella Unuscione -Carmilla Black -Carnage -Carnivore -Carolyn Parmenter -Carolyn Trainer -Carrie Alexander -Carrion -Carter Ghazikhanian -Cassandra Nova -Cassie Lang -Cassiopea -Cat -Cat-Man -Catiana -Cayman -Cecelia Reyes -Cecilia Reyes -Celestial Madonna -Centennial -Centurion -Centurious -Centurius -Cerberus -Cerebra -Cerise -Cessily Kincaid -Cethlann -Ch'od -Chaka -Challenger -Chamber -Chameleon -Champion of the Universe -Chan Luichow -Chance -Changeling -Chaos -Charcoal -Charles Xavier -Charlie-27 -Charon -Chase Stein -Cheetah -Chemistro -Chen Lu -Chi Demon -Chief Examiner -Chimera -Chloe Tran -Choice -Chondu the Mystic -Christopher Summers -Chrome -Chronos -Chthon -Chtylok -Citizen V -Claire Voyant -Claudette St. Croix -Clea -Clearcut -Cletus Kasady -Clint Barton -Clive -Cloak -Cloud -Cloud 9 -Clown -Coach -Coachwhip -Cobalt Man -Cobra -Cody Mushumanski Gun Man -Cold War -Coldblood -Coldfire -Collective Man -Collector -Colleen Wing -Colonel -Colonel America -Colossus -Comet -Comet Man -Commander Kraken -Commando -Conan the Barbarian -Condor -Conquer Lord -Conquest -Conquistador -Conrad Josten -Constrictor -Contemplator -Contessa -Contrary -Controller -Copperhead -Copycat -Coral -Cordelia Frost -Cornelius van Lunt -Corona -Corruptor -Corsair -Cottonmouth -Count Abyss -Count Nefaria -Courier -Cowgirl -Crazy Eight -Crime Master -Crime-Buster -Crimebuster -Crimson -Crimson Cavalier -Crimson Commando -Crimson Cowl -Crimson Craig -Crimson Daffodil -Crimson Dynamo -Crimson Dynamo V -Crimson and the Raven -Crippler -Crooked Man -Crossbones -Crossfire -Crown -Crucible -Crusader -Crusher -Crystal -Curtis Connors -Cutthroat -Cybele -Cybelle -Cyber -Cyborg X -Cyclone -Cyclops -Cypher -D'Ken -D'Spayre -D-Man -DJ -Dagger -Daisy Johnson -Dakimh the Enchanter -Dakota North -Damballah -Damion Hellstrom -Damon Dran -Dan Ketch -Danger -Daniel Rand -Danielle Moonstar -Dansen Macabre -Danvers Carol -Daredevil -Dark Angel -Dark Beast -Dark Phoenix -Dark-Crawler -Darkdevil -Darkhawk -Darkoth -Darkstar -David Cannon -Daytripper -Dazzler -Deacon Frost -Dead Girl -Deadhead -Deadly Ernest -Deadpool -Death -Death Adder -Death's Head -Death's Head II -Death-Stalker -Deathbird -Deathlok -Deathstroke -Deathurge -Deathwatch -Deborah Ritter -Debra Whitman -Decay -Decay II -Defensor -Delilah -Delphi -Delphine Courtney -Dementia -Demiurge -Demogoblin -Demogorge the God-Eater -Demolition Man -Derrick Slegers Speed -Desmond Pitt -Destiny -Destroyer -Destroyer of Demons -Devastator -Devil Dinosaur -Devil Hunter Gabriel -Devil-Slayer -Devos the Devastator -Diablo -Diamanda Nero -Diamond Lil -Diamondback -Diamondhead -Digitek -Dionysus -Dirtnap -Discus -Dittomaster -Dmitri Bukharin -Dmitri Smerdyakov -Doc Samson -Doctor Anthony Droom -Doctor Arthur Nagan -Doctor Bong -Doctor Demonicus -Doctor Doom -Doctor Dorcas -Doctor Droom -Doctor Druid -Doctor Faustus -Doctor Glitternight -Doctor Leery -Doctor Minerva -Doctor Octopus -Doctor Spectrum -Doctor Strange -Doctor Sun -Domina -Dominic Fortune -Dominic Petros -Domino -Dominus -Domo -Don Fortunato -Donald "Donny" Gill -Donald Pierce -Donald Ritter -Doop -Doorman -Doppelganger -Doppleganger -Dorma -Dormammu -Double Helix -Doug Ramsey -Doug and Jerry -Dougboy -Doughboy -Douglas Birely -Douglas Ramsey -Douglock -Dr. John Grey -Dr. Lemuel Dorcas -Dr. Marla Jameson -Dr. Otto Octavius -Dracula -Dragon Lord -Dragon Man -Dragon of the Moon -Dragoness -Dragonfly -Dragonwing -Drax the Destroyer -Dreadknight -Dreadnought -Dream Weaver -Dreaming Celestial -Dreamqueen -Dredmund Druid -Dromedan -Druid -Druig -Dum-Dum Dugan -Dusk -Dust -Dweller-in-Darkness -Dyna-Mite -Earth Lord -Earthquake -Ebenezer Laughton -Ebon Seeker -Echo -Ecstasy -Ectokid -Eddie Brock -Edward "Ned" Buckman -Edwin Jarvis -Eel -Egghead -Ego the Living Planet -El Aguila -El Muerto -Elaine Grey -Elathan -Electric Eve -Electro -ElectroCute -Electron -Eleggua -Elektra -Elektra Natchios -Elektro -Elf With A Gun -Elfqueen -Elias Bogan -Eliminator -Elixir -Elizabeth "Betsy" Braddock -Elizabeth Twoyoungmen -Ellie Phimster -Elsie-Dee -Elven -Elysius -Emil Blonsky -Emma Frost -Empath -Empathoid -Emplate -En Sabah Nur -Enchantress -Energizer -Enforcer -Enigma -Ent -Entropic Man -Eon -Epoch -Equilibrius -Equinox -Ereshkigal -Erg -Eric Slaughter -Eric Williams -Eric the Red -Erik Josten -Erik Killmonger -Erik Magnus Lehnsherr -Ernst -Eros -Eshu -Eson the Searcher -Eternal Brain -Eternity -Ethan Edwards -Eugene Judd -Ev Teel Urizen -Evangeline Whedon -Ever -Everett Thomas -Everyman -Evilhawk -Executioner -Exodus -Exploding Man -Exterminator -Ezekiel -Ezekiel Sims -Ezekiel Stane -Fabian Cortez -Fafnir -Fagin -Falcon -Fallen One -Famine -Fan Boy -Fandral -Fang -Fantasia -Fantastic Four -Fantomex -Farallah -Fasaud -Fashima -Fatale -Fateball -Father Time -Fault Zone -Fearmaster -Feedback -Felicia Hardy -Feline -Fenris -Fenris Wolf -Fer-de-Lance -Feral -Feron -Fever Pitch -Fight-Man -Fin -Fin Fang Foom -Firearm -Firebird -Firebolt -Firebrand -Firefrost -Firelord -Firepower -Firestar -Fixer -Fixx -Flag-Smasher -Flambe -Flash Thompson -Flatman -Flex -Flint Marko -Flubber -Fly -Flygirl -Flying Tiger -Foggy Nelson -Fontanelle -Foolkiller -Forbush Man -Force -Forearm -Foreigner -Forge -Forgotten One -Foxfire -Frank Castle -Frank Drake -Frank Payne -Frank Simpson -Frankenstein's Monster -Frankie Raye -Frankie and Victoria -Franklin Hall -Franklin Richards -Franklin Storm -Freak -Freak of Science -Freakmaster -Freakshow -Fred Myers -Frederick Slade -Free Spirit -Freedom Ring -Frenzy -Frey -Frigga -Frog-Man -Fury -Fusion -Futurist -G-Force -Gabe Jones -Gabriel Summers -Gabriel the Air-Walker -Gaea -Gaia -Gailyn Bailey -Galactus -Galaxy Master -Gambit -Gammenon the Gatherer -Gamora -Ganymede -Gardener -Gargantua -Gargantus -Gargouille -Gargoyle -Garokk the Petrified Man -Garrison Kane -Gatecrasher -Gateway -Gauntlet -Gavel -Gaza -Gazelle -Gazer -Geb -Gee -Geiger -Geirrodur -Gemini -General Orwell Taylor -Genis-Vell -George Stacy -George Tarleton -George Washington Bridge -Georgianna Castleberry -Gertrude Yorkes -Ghaur -Ghost -Ghost Dancer -Ghost Girl -Ghost Maker -Ghost Rider -Ghost Rider 2099 -Ghoul -Giant-Man -Gibbon -Gibborim -Gideon -Gideon Mace -Giganto -Gigantus -Gin Genie -Gladiator -Gladiatrix -Glamor -Glenn Talbot -Glitch -Glob -Glob Herman -Gloom -Glorian -Goblin Queen -Goblyn -Godfrey Calthrop -Gog -Goldbug -Golden Archer -Golden Girl -Golden Oldie -Goldeneye -Golem -Goliath -Gomi -Googam -Gorgeous George -Gorgilla -Gorgon -Gorilla Girl -Gorilla-Man -Gorr -Gosamyr -Grand Director -Grandmaster -Grappler -Grasshopper -Grasshopper II -Graviton -Gravity -Graydon Creed -Great Gambonnos -Great Video -Green Goblin -Green Goblin IV -Greer Grant -Greer Grant Nelson -Gregor Shapanka -Gregory Gideon -Gremlin -Grenade -Grey Gargoyle -Grey King -Griffin -Grim Hunter -Grim Reaper -Grizzly -Grog the God-Crusher -Gronk -Grotesk -Groundhog -Growing Man -Guardsman -Guido Carosella -Gunthar of Rigel -Gwen Stacy -Gypsy Moth -H.E.R.B.I.E. -Hack -Hag -Hairbag -Halflife -Halloween Jack -Hamilton Slade -Hammer Harrison -Hammer and Anvil -Hammerhead -Hangman -Hank McCoy -Hank Pym -Hanna Levy -Hannah Levy -Hannibal King -Harald Jaekelsson -Hardcase -Hardcore -Hardnose -Hardshell -Hardwire -Hargen the Measurer -Harmonica -Harness -Harold "Happy" Hogan -Harold H. Harold -Harpoon -Harpy -Harrier -Harry Leland -Harry Osborn -Hate-Monger -Haven -Havok -Hawkeye -Hawkeye II -Hawkshaw -Haywire -Hazard -Hazmat -Headknocker -Headlok -Heart Attack -Heather Cameron -Hebe -Hecate -Hector -Heimdall -Heinrich Zemo -Hela -Helio -Hellcat -Helleyes -Hellfire -Hellion -Hellrazor -Helmut Zemo -Henry "Hank" McCoy -Henry Peter Gyrich -Hensley Fargus -Hephaestus -Hepzibah -Her -Hera -Herbert Edgar Wyndham -Hercules -Herman Schultz -Hermes -Hermod -Hero -Hero for Hire -Herr Kleiser -Hideko Takata -High Evolutionary -High-Tech -Hijacker -Hildegarde -Him -Hindsight Lad -Hippolyta -Hisako Ichiki -Hit-Maker -Hitman -Hobgoblin -Hobgoblin II -Hoder -Hogun -Holly -Honcho -Honey Lemon -Hood -Hornet -Horus -Howard the Duck -Hrimhari -Hub -Hugh Jones -Hulk -Hulk 2099 -Hulkling -Human Cannonball -Human Fly -Human Robot -Human Top -Human Top II -Human Torch -Human Torch II -Humbug -Humus Sapien -Huntara -Hurricane -Husk -Hussar -Hybrid -Hybrid II -Hyde -Hydro -Hydro-Man -Hydron -Hyperion -Hyperkind -Hyperstorm -Hypnotia -Hyppokri -ISAAC -Icarus -Iceman -Icemaster -Idunn -Iguana -Ikaris -Ikonn -Ikthalon -Illusion -Illyana Rasputin -Immortus -Impala -Imperial Hydra -Impossible Man -Impulse -In-Betweener -Indech -Indra -Inertia -Infamnia -Infant Terrible -Infectia -Inferno -Infinity -Interloper -Invisible Girl -Invisible Woman -Inza -Ion -Iridia -Iron Cross -Iron Fist -Iron Lad -Iron Maiden -Iron Man -Iron Man 2020 -Iron Monger -Ironclad -Isaac Christians -Isaiah Bradley -Isbisa -Isis -Ivan Kragoff -J. Jonah Jameson -J2 -Jack Flag -Jack Frost -Jack Kirby -Jack O'Lantern -Jack Power -Jack of Hearts -Jack-in-the-Box -Jackal -Jackdaw -Jackhammer -Jackpot -Jackson Arvad -Jacob "Jake" Fury -Jacqueline Falsworth -Jacques DuQuesne -Jade Dragon -Jaeger -Jaguar -Jamal Afari -James "Jimmy" Marks -James Dr. Power -James Howlett -James Jaspers -James Madrox -James Proudstar -James Rhodes -James Sanders -Jamie Braddock -Jane Foster -Jane Kincaid -Janet van Dyne -Jann -Janus -Jared Corbo -Jarella -Jaren -Jason -Jawynn Dueck the Iron Christian of Faith -Jazz -Jean DeWolff -Jean Grey -Jean Grey-Summers -Jean-Paul Beaubier -Jeanne-Marie Beaubier -Jebediah Guthrie -Jeffrey Mace -Jekyll -Jennifer Kale -Jennifer Walters -Jens Meilleur Slap Shot -Jericho Drumm -Jerome Beechman -Jerry Jaxon -Jessica Drew -Jessica Jones -Jester -Jigsaw -Jim Hammond -Jimaine Szardos -Jimmy Woo -Jocasta -Joe Cartelli -Joe Fixit -Joey Bailey -Johann Schmidt -John Doe -John Falsworth -John Jameson -John Proudstar -John Ryker -John Sublime -John Walker -Johnny Blaze -Johnny Ohm -Johnny Storm -Jolt -Jon Spectre -Jonas Harrow -Jonathan "John" Garrett -Jonathan Richards -Jonothon Starsmore -Jordan Seberius -Joseph -Joshua Guthrie -Joystick -Jubilee -Judas Traveller -Jude the Entropic Man -Juggernaut -Julie Power -Jumbo Carnation -Junkpile -Junta -Justice -Justin Hammer -Justine Hammer -Ka-Zar -Kaine -Kala -Kaluu -Kamal -Kamo Tharnn -Kamuu -Kang the Conqueror -Kangaroo -Karen Page -Karima Shapandar -Karkas -Karl Lykos -Karl Malus -Karl Mordo -Karla Sofen -Karma -Karnak -Karnilla -Karolina Dean -Karthon the Quester -Kasper Cole -Kate Bishop -Kate Neville -Katherine "Kitty" Pryde -Katherine Reynolds -Katie Power -Katrina Luisa van Horne -Katu -Keen Marlow -Kehl of Tauran -Keith Kilham -Kem Horkus -Kenneth Crichton -Key -Khaos -Khonshu -Khoryphos -Kiber the Cruel -Kick-Ass -Kid Colt -Kid Nova -Kiden Nixon -Kierrok -Killer Shrike -Killpower -Killraven -Kilmer -Kimura -King Bedlam -Kingo Sunen -Kingpin -Kirigi -Kirtsyn Perrin Short Stop -Kismet -Kismet Deadly -Kiss -Kiwi Black -Kkallakku -Kl'rt -Klaatu -Klaw -Kleinstocks -Knickknack -Kofi Whitemane -Kogar -Kohl Harder Boulder Man -Korath the Pursuer -Korg -Kormok -Korrek -Korvac -Korvus -Kosmos -Kraken -Krakkan -Krang -Kraven the Hunter -Krista Marwan -Kristoff Vernard -Kristoff von Doom -Kro -Krystalin -Kubik -Kukulcan -Kurse -Kurt Wagner -Kwannon -Kyle Gibney -Kylun -Kymaera -La Lunatica -La Nuit -Lacuna -Lady Deathstrike -Lady Jacqueline Falsworth Crichton -Lady Killer -Lady Lark -Lady Lotus -Lady Mandarin -Lady Mastermind -Lady Octopus -Lament -Lancer -Landslide -Larry Bodine -Lasher -Laura Dean -Layla Miller -Lazarus -Leader -Leap-Frog -Leash -Lee Forrester -Leech -Left Hand -Left-Winger -Legacy -Legion -Leila Davis -Leir -Lemuel Dorcas -Leo -Leonard Samson -Leonus -Letha -Levan -Lianda -Libra -Lifeforce -Lifeguard -Lifter -Lightbright -Lighting Rod -Lightmaster -Lightspeed -Lila Cheney -Lilandra Neramani -Lilith, the Daughter of Dracula -Lin Sun -Link -Lionheart -Live Wire -Living Brain -Living Colossus -Living Diamond -Living Eraser -Living Hulk -Living Laser -Living Lightning -Living Monolith -Living Mummy -Living Pharaoh -Living Planet -Living Totem -Living Tribunal -Liz Allan -Lizard -Llan the Sorcerer -Lloigoroth -Llyra -Llyron -Loa -Lockdown -Lockheed -Lockjaw -Locksmith -Locus -Locust -Lodestone -Logan -Loki -Longneck -Longshot -Lonnie Thompson Lincoln -Looter -Lord Chaos -Lord Dark Wind -Lord Pumpkin -Lorelei -Lorelei II -Lorelei Travis -Lorna Dane -Lorvex -Loss -Louise Mason -Lucas Brand -Luchino Nefaria -Lucifer -Ludi -Luke Cage -Luna -Lunatica -Lunatik -Lupa -Lupo -Lurking Unknown -Lyja -Lynx -M -M-Twins -MN-E (Ultraverse) -MODAM -MODOK -Mac Gargan -Mach-IV -Machine Man -Machine Teen -Machinesmith -Mad Dog Rassitano -Mad Jack -Mad Jim Jaspers -Mad Thinker -Mad Thinker’s Awesome Android -Mad-Dog -Madam Slay -Madame Hydra -Madame MacEvil -Madame Masque -Madame Menace -Madame Web -Madcap -Madeline Joyce -Madelyne Pryor -Madison Jeffries -Maelstrom -Maestro -Magdalena -Magdalene -Maggott -Magician -Magik -Magilla -Magma -Magneto -Magnum -Magnus -Magus -Maha Yogi -Mahkizmo -Major Mapleleaf -Makkari -Malekith the Accursed -Malice -Mammomax -Man Mountain Marko -Man-Ape -Man-Beast -Man-Brute -Man-Bull -Man-Eater -Man-Elephant -Man-Killer -Man-Spider -Man-Thing -Man-Wolf -Manbot -Mandarin -Mandrill -Mandroid -Mangle -Mangog -Manikin -Manslaughter -Manta -Mantis -Mantra -Mar-Vell -Marc Spector -Marduk Kurios -Margali Szardos -Margaret Power -Margo Damian -Maria Hill -Mariko Yashida -Marius St. Croix -Mark Gervaisnight Shade -Mark Raxton -Mark Scarlotti -Mark Todd -Marlene Alraune -Marrina -Marrina Smallwood -Marrow -Marsha Rosenberg -Martha Johansson -Martin Gold -Martin Preston -Martinex -Marvel Boy -Marvel Girl -Marvel Man -Marvin Flumm -Mary "Skeeter" MacPherran -Mary Jane Parker -Mary Jane Watson -Mary Walker -Mary Zero -Masked Marauder -Masked Marvel -Masked Rose -Masque -Mass Master -Master Khan -Master Man -Master Menace -Master Mold -Master Order -Master Pandemonium -Master of Vengeance -Mastermind -Mastermind of the UK -Matador -Match -Matsu'o Tsurayaba -Matt Murdock -Mauler -Maur-Konn -Mauvais -Maverick -Max -Maxam -Maximus -Maxwell Dillon -May "Mayday" Parker -May Parker -Mayhem -Maynard Tiboldt -Meanstreak -Meathook -Mechamage -Medusa -Meggan -Meggan Braddock -Mekano -Meld -Melee -Melissa Gold -Melody Guthrie -Meltdown -Melter -Mentallo -Mentor -Mentus -Mephisto -Mercurio -Mercury -Mercy -Merlin -Mesmero -Metal Master -Metalhead -Meteor Man -Meteorite -Meteorite II -Michael Nowman -Michael Twoyoungmen -Micro -Microchip -Micromax -Midas -Midgard Serpent -Midnight -Midnight Man -Midnight Sun -Miek -Miguel Espinosa -Miguel O'Hara -Miguel Santos -Mikado -Mikey -Mikhail Rasputin -Mikula Golubev -Milan -Miles Warren -Milos Masaryk -Mimic -Mimir -Mindmeld -Mindworm -Miracle Man -Mirage -Mirage II -Misfit -Miss America -Missing Link -Mist Mistress -Mister Buda -Mister Doll -Mister Fear -Mister Hyde -Mister Jip -Mister Machine -Mister One -Mister Sensitive -Mister Sinister -Mister Two -Mister X -Misty Knight -Mockingbird -Modred the Mystic -Mogul of the Mystic Mountain -Moira Brandon -Moira MacTaggert -Mojo -Mole Man -Molecule Man -Molly Hayes -Molten Man -Mondo -Monet St. Croix -Mongoose -Monica Rappaccini -Monsoon -Monstra -Monstro the Mighty -Moon Knight -Moon-Boy -Moondark -Moondragon -Moonhunter -Moonstone -Mop Man -Morbius -Mordred -Morg -Morgan Le Fay -Morlun -Morning Star -Morph -Morpheus -Morris Bench -Mortimer Toynbee -Moses Magnum -Mosha -Mother Earth -Mother Nature -Mother Night -Mother Superior -Motormouth -Mountjoy -Mr. Fish -Mr. Justice -Mr. M -Mr. Wu -Ms. MODOK -Ms. Marvel -Ms. Steed -Multiple Man -Murmur -Murmur II -Mutant Master -Mutant X -Myron MacLain -Mys-Tech -Mysterio -Mystique -N'Gabthoth -N'Garai -N'astirh -NFL Superpro -Naga -Nameless One -Namor McKenzie -Namor the Sub-Mariner -Namora -Namorita -Nanny -Nate Grey -Nathaniel Essex -Nathaniel Richards -Native -Nebula -Nebulo -Nebulon -Nebulos -Necrodamus -Necromantra -Ned Horrocks -Ned Leeds -Needle -Nefarius -Negasonic Teenage Warhead -Nekra -Nekra Sinclar -Nemesis -Neophyte -Neptune -Network -Neuronne -Neurotap -New Goblin -Nezarr the Calculator -Nicholas Maunder -Nicholas Scratch -Nick Fury -Nico Minoru -Nicole St. Croix -Night Nurse -Night Rider -Night Thrasher -Nightcrawler -Nighthawk -Nightmare -Nightshade -Nightside -Nightwatch -Nightwind -Nikki -Niles Van Roekel -Nimrod -Ningal -Nitro -Nobilus -Nocturne -Noh-Varr -Nomad -Norman Osborn -Norns -Norrin Radd -Northstar -Nosferata -Nova -Nova-Prime -Novs -Nox -Nth Man -Nth Man: the Ultimate Ninja -Nuke - Frank Simpson -Nuke - Squadron Supreme Member -Nuklo -Numinus -Nut -Obadiah Stane -Obituary -Obliterator -Oblivion -Occulus -Ocean -Ocelot -Oddball -Odin -Ogre -Ogress -Omega -Omega Red -Omega the Unknown -Omen -Omerta -One Above All -Oneg the Prober -Onslaught -Onyxx -Ooze -Optoman -Oracle -Orator -Orb -Orbit -Orchid -Ord -Order -Orikal -Orka -Ororo Munroe -Orphan -Orphan-Maker -Osiris -Outlaw -Outrage -Overkill -Overmind -Overrider -Owl -Ox -Ozone -Ozymandias -Paibo -Paige Guthrie -Paladin -Paradigm -Paragon -Paralyzer -Paris -Pasco -Paste-Pot Pete -Patch -Pathway -Patriot -Patriot II -Patsy Hellstrom -Patsy Walker -Paul Bailey -Paul Norbert Ebersol -Paul Patterson -Payback -Peace Monger -Peepers -Peggy Carter -Penance -Penance II -Peregrine -Perfection -Perseus -Persuader -Persuasion -Perun -Pete Wisdom -Peter Criss -Peter Noble -Peter Parker -Peter Petruski -Phade -Phage -Phalanx -Phantazia -Phantom Blonde -Phantom Eagle -Phantom Rider -Phastos -Phat -Phil Urich -Philip Fetter -Phineas T. Horton -Phoenix -Photon -Phyla-Vell -Pietro Maximoff -Piledriver -Piotr Rasputin -Pip the Troll -Pipeline -Piper -Piranha -Pisces -Pistol -Pixie -Pixx -Plague -Plantman -Plasma -Plazm -Plug -Plunderer -Pluto -Poison -Polaris -Poltergeist -Porcupine -Portal -Possessor -Postman -Postmortem -Poundcakes -Powderkeg -Power Broker -Power Man -Power Princess -Power Skrull -Powerhouse -Powerpax -Presence -Pressure -Prester John -Pretty Persuasions -Preview -Primal -Prime -Prime Mover -Primevil -Primus -Princess Python -Proctor -Prodigy -Professor Power -Professor X -Projector -Prometheus -Protector -Proteus -Prototype -Prowler -Psi-Lord -Psyche -Psycho-Man -Psyklop -Psylocke -Puck -Puff Adder -Puishannt -Pulse -Puma -Punchout -Punisher -Punisher 2099 -Puppet Master -Purge -Purple Girl -Purple Man -Pyre -Pyro -Quagmire -Quantum -Quasar -Quasar II -Quasimodo -Quentin Beck -Quentin Quire -Quicksand -Quicksilver -Quincy Harker -Raa of the Caves -Rachel Grey -Rachel Summers -Rachel van Helsing -Radian -Radioactive Man -Radion the Atomic Man -Radius -Rafferty -Rage -Raggadorr -Rahne Sinclair -Rainbow -Rama-Tut -Raman -Ramrod -Ramshot -Rancor -Randall Shire -Random -Ranger -Ransak the Reject -Rattler -Ravage 2099 -Raving Beauty -Rawhide Kid -Rax -Raymond Sikorsky -Raza -Razor Fist -Razorback -Reaper -Rebel -Recorder -Red Claw -Red Ghost -Red Guardian -Red Lotus -Red Nine -Red Raven -Red Ronin -Red Shift -Red Skull -Red Skull II -Red Wolf -Redeemer -Redneck -Redwing -Reeva Payge -Reignfire -Reject -Remnant -Remy LeBeau -Reptyl -Revanche -Rex Mundi -Rhiannon -Rhino -Ricadonna -Richard Fisk -Richard Parker -Richard Rider -Rick Jones -Ricochet -Rictor -Rigellian Recorder -Right-Winger -Ringer -Ringleader -Ringmaster -Ringo Kid -Rintrah -Riot -Riot Grrl -Ripfire -Ritchie Gilmore -Rl'nnd -Robbie Robertson -Robert "Bobby" Drake -Robert Bruce Banner -Robert Hunter -Robert Kelly -Robert da Costa -Rock -Rock Python -Rocket Raccoon -Rocket Racer -Rodstvow -Rogue -Rom the Spaceknight -Roma -Romany Wisdom -Ronan the Accuser -Rose -Roughhouse -Roulette -Royal Roy -Ruby Thursday -Ruckus -Rumiko Fujikawa -Rune -Runner -Rush -Rusty Collins -Ruth Bat-Seraph -Ryder -S'byll -S'ym -Sabra -Sabreclaw -Sabretooth -Sack -Sage -Sagittarius -Saint Anna -Saint Elmo -Sally Blevins -Sally Floyd -Salvo -Sam Sawyer -Sam Wilson -Samuel "Starr" Saxon -Samuel Guthrie -Samuel Silke -Samuel Smithers -Sandman -Sangre -Sara Grey -Sasquatch -Satana -Satannish -Saturnyne -Sauron -Savage Steel -Sayge -Scaleface -Scalphunter -Scanner -Scarecrow -Scarecrow II -Scarlet Beetle -Scarlet Centurion -Scarlet Scarab -Scarlet Spider -Scarlet Spiders -Scarlet Witch -Schemer -Scimitar -Scintilla -Scorcher -Scorpia -Scorpio -Scorpion -Scott Summers -Scott Washington -Scourge of the Underworld -Scrambler -Scream -Screaming Mimi -Screech -Scrier -Sea Urchin -Seamus Mellencamp -Sean Cassidy -Sean Garrison -Sebastian Shaw -Seeker -Sekhmet -Selene -Senator Robert Kelly -Senor Muerte -Sentry -Sepulchre -Sergeant Fury -Sergei Kravinoff -Serpentina -Sersi -Set -Seth -Shadow King -Shadow Slasher -Shadow-Hunter -Shadowcat -Shadowmage -Shadrac -Shalla-Bal -Shaman -Shamrock -Shang-Chi -Shanga -Shanna the She-Devil -Shaper of Worlds -Shard -Sharon Carter -Sharon Friedlander -Sharon Ventura -Shathra -Shatter -Shatterfist -Shatterstar -She-Hulk -She-Thing -She-Venom -Shellshock -Shen Kuei -Shi'ar Gladiator -Shinchuko Lotus -Shingen Harada -Shinobi Shaw -Shirow Ishihara -Shiva -Shiver Man -Shocker -Shockwave -Shola Inkosi -Shooting Star -Shotgun -Shriek -Shriker -Shroud -Shrunken Bones -Shuma-Gorath -Sidewinder -Siege -Siena Blaze -Sif -Sigmar -Sigyn -Sikorsky -Silhouette -Silly Seal -Silver -Silver Dagger -Silver Fox -Silver Sable -Silver Samurai -Silver Scorpion -Silver Squire -Silver Surfer -Silverclaw -Silvermane -Simon Williams -Sin -Sin-Eater -Sinister -Sir Steel -Siryn -Sise-Neg -Skein -Skids -Skin -Skinhead -Skull the Slayer -Skullcrusher -Skullfire -Skunge the Laxidazian Troll -Skyhawk -Skywalker -Slab -Slapstick -Sleek -Sleeper -Sleepwalker -Slick -Sligguth -Slipstream -Slither -Sludge -Slug -Sluggo -Sluk -Slyde -Smart Alec -Smartship Friday -Smasher -Smuggler -Smuggler II -Snowbird -Snowfall -Solara -Solarman -Solarr -Soldier X -Solitaire -Solo -Solomon O'Sullivan -Son of Satan -Songbird -Soulfire -Space Phantom -Space Turnip -Specialist -Spectra -Spectral -Speed -Speed Demon -Speedball -Speedo -Spellbinder -Spellcheck -Spencer Smythe -Sphinx -Sphinxor -Spider Doppelganger -Spider-Girl -Spider-Ham -Spider-Man -Spider-Slayer -Spider-Woman -Spidercide -Spike -Spike Freeman -Spinnerette -Spiral -Spirit of '76 -Spitfire -Spoilsport -Spoor -Spot -Sprite -Sputnik -Spyder -Spymaster -Spyne -Squidboy -Squirrel Girl -St. John Allerdyce -Stacy X -Stained Glass Scarlet -Stakar -Stallior -Stanley Stewart -Star Stalker -Star Thief -Star-Dancer -Star-Lord -Starbolt -Stardust -Starfox -Starhawk -Starlight -Starr the Slayer -Starshine -Starstreak -Stature -Steel Raven -Steel Serpent -Steel Spider -Stegron -Stellaris -Stem Cell -Stentor -Stephen Colbert -Stephen Strange -Steve Rogers -Steven Lang -Stevie Hunter -Stick -Stiletto -Stilt-Man -Stinger -Stingray -Stitch -Stone -Stonecutter -Stonewall -Storm -Stranger -Stratosfire -Straw Man -Strobe -Strong Guy -Strongarm -Stryfe -Stunner -Stuntmaster -Stygorr -Stygyro -Styx and Stone -Sub-Mariner -Sugar Man -Suicide -Sultan -Sun Girl -Sunder -Sundragon -Sunfire -Sunpyre -Sunset Bain -Sunspot -Sunstreak -Sunstroke -Sunturion -Super Rabbit -Super Sabre -Super-Adaptoid -Super-Nova -Super-Skrull -SuperPro -Supercharger -Superia -Supernalia -Suprema -Supreme Intelligence -Supremor -Surge -Surtur -Susan Richards -Susan Storm -Sushi -Svarog -Swarm -Sweetface -Swordsman -Sybil Dorn -Sybil Dvorak -Synch -T-Ray -Tabitha Smith -Tag -Tagak the Leopard Lord -Tailhook -Taj Nital -Talia Josephine Wagner -Talisman -Tamara Rahn -Tana Nile -Tantra -Tanya Anderssen -Tarantula -Tarot -Tartarus -Taskmaster -Tatterdemalion -Tattletale -Tattoo -Taurus -Techno -Tefral the Surveyor -Tempest -Tempo -Tempus -Temugin -Tenpin -Termagaira -Terminator -Terminatrix -Terminus -Terrax the Tamer -Terraxia -Terror -Tess-One -Tessa -Tether -Tethlam -Tex Dawson -Texas Twister -Thakos -Thane Ector -Thanos -The Amazing Tanwir Ahmed -The Angel -The Blank -The Destroyer -The Entity -The Grip -The Night Man -The Profile -The Russian -The Stepford Cuckoos -The Symbiote -The Wink -Thena -Theresa Cassidy -Thermo -Thin Man -Thing -Thinker -Thirty-Three -Thog -Thomas Halloway -Thor -Thor Girl -Thornn -Threnody -Thumbelina -Thunderball -Thunderbird -Thunderbolt -Thunderclap -Thunderfist -Thunderstrike -Thundra -Tiboro -Tiger Shark -Tigra -Timberius -Time Bomb -Timeshadow -Timeslip -Tinkerer -Titan -Titania -Titanium Man -Tito Bohusk -Toad -Toad-In-Waiting -Todd Arliss -Tom Cassidy -Tom Corsi -Tom Foster -Tom Thumb -Tomazooma -Tombstone -Tommy -Tommy Lightning -Tomorrow Man -Tony Stark -Topaz -Topspin -Torgo of Mekka -Torgo the Vampire -Toro -Torpedo -Torrent -Torso -Tower -Toxin -Trader -Trapper -Trapster -Tremolo -Trevor Fitzroy -Tri-Man -Triathlon -Trick Shot -Trioccula -Trip Monroe -Triton -Troll -Trump -Tuc -Tugun -Tumbler -Tundra -Turac -Turbo -Turner Century -Turner D. Century -Tusk -Tutinax the Mountain-Mover -Two-Gun Kid -Tyger Tiger -Typeface -Typhoid -Typhoid Mary -Typhon -Tyr -Tyrak -Tyrannosaur -Tyrannus -Tyrant -Tzabaoth -U-Go Girl -U-Man -USAgent -Uatu -Ulik -Ultimo -Ultimus -Ultra-Marine -Ultragirl -Ultron -Ulysses -Umar -Umbo -Uncle Ben Parker -Uni-Mind -Unicorn -Union Jack -Unseen -Unthinnk -Unus the Untouchable -Unuscione -Ursa Major -Urthona -Utgard-Loki -Vagabond -Vague -Vakume -Valentina Allegra de La Fontaine -Valerie Cooper -Valinor -Valkin -Valkyrie -Valtorr -Vamp -Vampire by Night -Vance Astro -Vance Astrovik -Vanguard -Vanisher -Vapor -Vargas -Varnae -Vashti -Vavavoom -Vector -Vegas -Veil -Vengeance -Venom -Venomm -Venus -Venus Dee Milo -Veritas -Vermin -Vertigo -Vesta -Vibraxas -Vibro -Victor Creed -Victor Mancha -Victor Strange -Victor von Doom -Victorius -Vidar -Vincente -Vindaloo -Vindicator -Viper -Virako -Virginia "Pepper" Potts -Virgo -Vishanti -Visimajoris -Vision -Vivisector -Vixen -Volcana -Volla -Volpan -Volstagg -Vulcan -Vulture -Wade Wilson -Wallflower -Walter Newell -Wanda Maximoff -War -War Eagle -War Machine -War V -Warbird -Warhawk -Warlock -Warpath -Warren III Worthington -Warrior Woman -Warstar -Warstrike -Warwolves -Washout -Wasp -Watcher -Water Wizard -Watoomb -Weapon X -Wendell Vaughn -Wendigo -Werewolf by Night -Western Kid -Whiplash -Whirlwind -Whistler -White Fang -White Pilgrim -White Queen -White Rabbit -White Tiger -Whiteout -Whizzer -Wiccan -Wicked -Widget -Wilbur Day -Wild Child -Wild Thing -Wildboys -Wildpride -Wildside -Will o' the Wisp -William Baker -William Stryker -Willie Lumpkin -Wilson Fisk -Wind Dancer -Wind Warrior -Windeagle -Windshear -Winky Man -Winter Soldier -Witchfire -Wiz Kid -Wizard -Wolf -Wolfsbane -Wolverine -Wonder Man -Wong -Woodgod -Worm -Wraith -Wrath -Wreckage -Wrecker -Wundarr the Aquarian -Wyatt Wingfoot -Wysper -X-23 -X-Cutioner -X-Man -X-Ray -X-Treme -Xandu -Xavin -Xemnu the Titan -Xemu -Xi'an Chi Xan -Xorn -Xorr the God-Jewel -Y'Garon -Yandroth -Yellow Claw -Yellowjacket -Yeti -Yith -Ymir -Yondu -Yrial -Yukio -Yukon Jack -Yuri Topolov -Yuriko Oyama -Zabu -Zach -Zaladane -Zarathos -Zarek -Zartra -Zebediah Killgrave -Zeitgeist -Zero -Zero-G -Zeus -Ziggy Pig -Zip-Zap -Zodiak -Zom -Zombie -Zuras -Zzzax -gen Harada -the Living Colossus It -the Living Darkness Null -the Renegade Watcher Aron -the Tomorrow Man Zarrko diff --git a/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java new file mode 100644 index 00000000000..e652a670a67 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action; + +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +public class DocWriteResponseTests extends ESTestCase { + public void testGetLocation() { + DocWriteResponse response = new DocWriteResponse(new ShardId("index", "uuid", 0), "type", "id", 0) { + // DocWriteResponse is abstract so we have to sneak a subclass in here to test it. + }; + assertEquals("/index/type/id", response.getLocation(null)); + assertEquals("/index/type/id?routing=test_routing", response.getLocation("test_routing")); + } +} diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java similarity index 71% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java rename to core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index cda5ba2161e..5007d5c830f 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -17,13 +17,9 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.action.bulk; import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; @@ -38,17 +34,21 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.script.ScriptException; import org.elasticsearch.test.ESIntegTestCase; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CyclicBarrier; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; @@ -58,10 +58,56 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class BulkTests extends ESIntegTestCase { +public class BulkWithUpdatesIT extends ESIntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("script.default_lang", CustomScriptPlugin.NAME) + .build(); + } + @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + + scripts.put("ctx._source.field += 1", vars -> srcScript(vars, source -> { + Integer field = (Integer) source.get("field"); + return source.replace("field", field + 1); + })); + + scripts.put("ctx._source.counter += 1", vars -> srcScript(vars, source -> { + Integer counter = (Integer) source.get("counter"); + return source.replace("counter", counter + 1); + })); + + scripts.put("ctx._source.field2 = 'value2'", vars -> srcScript(vars, source -> source.replace("field2", "value2"))); + + scripts.put("throw script exception on unknown var", vars -> { + throw new ScriptException("message", null, Collections.emptyList(), "exception on unknown var", CustomScriptPlugin.NAME); + }); + + scripts.put("ctx.op = \"none\"", vars -> ((Map) vars.get("ctx")).put("op", "none")); + scripts.put("ctx.op = \"delete\"", vars -> ((Map) vars.get("ctx")).put("op", "delete")); + return scripts; + } + + @SuppressWarnings("unchecked") + static Object srcScript(Map vars, Function, Object> f) { + Map ctx = (Map) vars.get("ctx"); + + Map source = (Map) ctx.get("_source"); + return f.apply(source); + } } public void testBulkUpdateSimple() throws Exception { @@ -82,25 +128,26 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkItemResponse.getIndex(), equalTo("test")); } + final Script script = new Script("ctx._source.field += 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + bulkResponse = client().prepareBulk() - .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("1") - .setScript(new Script("ctx._source.field += 1", ScriptService.ScriptType.INLINE, null, null))) - .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("2") - .setScript(new Script("ctx._source.field += 1", ScriptService.ScriptType.INLINE, null, null)).setRetryOnConflict(3)) + .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("1").setScript(script)) + .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("2").setScript(script).setRetryOnConflict(3)) .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("3") - .setDoc(jsonBuilder().startObject().field("field1", "test").endObject())).execute().actionGet(); + .setDoc(jsonBuilder().startObject().field("field1", "test").endObject())) + .get(); assertThat(bulkResponse.hasFailures(), equalTo(false)); assertThat(bulkResponse.getItems().length, equalTo(3)); for (BulkItemResponse bulkItemResponse : bulkResponse) { assertThat(bulkItemResponse.getIndex(), equalTo("test")); } - assertThat(((UpdateResponse) bulkResponse.getItems()[0].getResponse()).getId(), equalTo("1")); - assertThat(((UpdateResponse) bulkResponse.getItems()[0].getResponse()).getVersion(), equalTo(2L)); - assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getId(), equalTo("2")); - assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(2L)); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getId(), equalTo("3")); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(2L)); + assertThat(bulkResponse.getItems()[0].getResponse().getId(), equalTo("1")); + assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(2L)); + assertThat(bulkResponse.getItems()[1].getResponse().getId(), equalTo("2")); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L)); + assertThat(bulkResponse.getItems()[2].getResponse().getId(), equalTo("3")); + assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L)); GetResponse getResponse = client().prepareGet().setIndex("test").setType("type1").setId("1").setFields("field").execute() .actionGet(); @@ -120,26 +167,23 @@ public class BulkTests extends ESIntegTestCase { bulkResponse = client() .prepareBulk() - .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("6") - .setScript(new Script("ctx._source.field += 1", ScriptService.ScriptType.INLINE, null, null)) + .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("6").setScript(script) .setUpsert(jsonBuilder().startObject().field("field", 0).endObject())) - .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("7") - .setScript(new Script("ctx._source.field += 1", ScriptService.ScriptType.INLINE, null, null))) - .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("2") - .setScript(new Script("ctx._source.field += 1", ScriptService.ScriptType.INLINE, null, null))).execute() - .actionGet(); + .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("7").setScript(script)) + .add(client().prepareUpdate().setIndex(indexOrAlias()).setType("type1").setId("2").setScript(script)) + .get(); assertThat(bulkResponse.hasFailures(), equalTo(true)); assertThat(bulkResponse.getItems().length, equalTo(3)); - assertThat(((UpdateResponse) bulkResponse.getItems()[0].getResponse()).getId(), equalTo("6")); - assertThat(((UpdateResponse) bulkResponse.getItems()[0].getResponse()).getVersion(), equalTo(1L)); + assertThat(bulkResponse.getItems()[0].getResponse().getId(), equalTo("6")); + assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); assertThat(bulkResponse.getItems()[1].getResponse(), nullValue()); assertThat(bulkResponse.getItems()[1].getFailure().getIndex(), equalTo("test")); assertThat(bulkResponse.getItems()[1].getFailure().getId(), equalTo("7")); assertThat(bulkResponse.getItems()[1].getFailure().getMessage(), containsString("document missing")); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getId(), equalTo("2")); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getIndex(), equalTo("test")); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(3L)); + assertThat(bulkResponse.getItems()[2].getResponse().getId(), equalTo("2")); + assertThat(bulkResponse.getItems()[2].getResponse().getIndex(), equalTo("test")); + assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(3L)); getResponse = client().prepareGet().setIndex("test").setType("type1").setId("6").setFields("field").execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); @@ -164,11 +208,11 @@ public class BulkTests extends ESIntegTestCase { .add(client().prepareIndex("test", "type", "1").setSource("field", "2")).get(); assertTrue(((IndexResponse) bulkResponse.getItems()[0].getResponse()).isCreated()); - assertThat(((IndexResponse) bulkResponse.getItems()[0].getResponse()).getVersion(), equalTo(1L)); + assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); assertTrue(((IndexResponse) bulkResponse.getItems()[1].getResponse()).isCreated()); - assertThat(((IndexResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(1L)); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(1L)); assertFalse(((IndexResponse) bulkResponse.getItems()[2].getResponse()).isCreated()); - assertThat(((IndexResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(2L)); + assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L)); bulkResponse = client().prepareBulk() .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc("field", "2")) @@ -176,29 +220,37 @@ public class BulkTests extends ESIntegTestCase { .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc("field", "3")).get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); - assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(2L)); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(3L)); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L)); + assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(3L)); bulkResponse = client().prepareBulk() - .add(client().prepareIndex("test", "type", "e1").setSource("field", "1").setVersion(10).setVersionType(VersionType.EXTERNAL)) - .add(client().prepareIndex("test", "type", "e2").setSource("field", "1").setVersion(10).setVersionType(VersionType.EXTERNAL)) - .add(client().prepareIndex("test", "type", "e1").setSource("field", "2").setVersion(12).setVersionType(VersionType.EXTERNAL)).get(); + .add(client().prepareIndex("test", "type", "e1") + .setSource("field", "1").setVersion(10).setVersionType(VersionType.EXTERNAL)) + .add(client().prepareIndex("test", "type", "e2") + .setSource("field", "1").setVersion(10).setVersionType(VersionType.EXTERNAL)) + .add(client().prepareIndex("test", "type", "e1") + .setSource("field", "2").setVersion(12).setVersionType(VersionType.EXTERNAL)) + .get(); assertTrue(((IndexResponse) bulkResponse.getItems()[0].getResponse()).isCreated()); - assertThat(((IndexResponse) bulkResponse.getItems()[0].getResponse()).getVersion(), equalTo(10L)); + assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(10L)); assertTrue(((IndexResponse) bulkResponse.getItems()[1].getResponse()).isCreated()); - assertThat(((IndexResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(10L)); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(10L)); assertFalse(((IndexResponse) bulkResponse.getItems()[2].getResponse()).isCreated()); - assertThat(((IndexResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(12L)); + assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(12L)); bulkResponse = client().prepareBulk() - .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "2").setVersion(10)) // INTERNAL - .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "3").setVersion(20).setVersionType(VersionType.FORCE)) - .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "4").setVersion(20).setVersionType(VersionType.INTERNAL)).get(); + .add(client().prepareUpdate("test", "type", "e1") + .setDoc("field", "2").setVersion(10)) // INTERNAL + .add(client().prepareUpdate("test", "type", "e1") + .setDoc("field", "3").setVersion(20).setVersionType(VersionType.FORCE)) + .add(client().prepareUpdate("test", "type", "e1") + .setDoc("field", "4").setVersion(20).setVersionType(VersionType.INTERNAL)) + .get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); - assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(20L)); - assertThat(((UpdateResponse) bulkResponse.getItems()[2].getResponse()).getVersion(), equalTo(21L)); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(20L)); + assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(21L)); } public void testBulkUpdateMalformedScripts() throws Exception { @@ -215,12 +267,12 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkResponse.getItems().length, equalTo(3)); bulkResponse = client().prepareBulk() - .add(client().prepareUpdate().setIndex("test").setType("type1").setId("1") - .setScript(new Script("ctx._source.field += a", ScriptService.ScriptType.INLINE, null, null)).setFields("field")) - .add(client().prepareUpdate().setIndex("test").setType("type1").setId("2") - .setScript(new Script("ctx._source.field += 1", ScriptService.ScriptType.INLINE, null, null)).setFields("field")) - .add(client().prepareUpdate().setIndex("test").setType("type1").setId("3") - .setScript(new Script("ctx._source.field += a", ScriptService.ScriptType.INLINE, null, null)).setFields("field")) + .add(client().prepareUpdate().setIndex("test").setType("type1").setId("1").setFields("field") + .setScript(new Script("throw script exception on unknown var", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .add(client().prepareUpdate().setIndex("test").setType("type1").setId("2").setFields("field") + .setScript(new Script("ctx._source.field += 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .add(client().prepareUpdate().setIndex("test").setType("type1").setId("3").setFields("field") + .setScript(new Script("throw script exception on unknown var", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) .execute().actionGet(); assertThat(bulkResponse.hasFailures(), equalTo(true)); @@ -229,10 +281,9 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkResponse.getItems()[0].getFailure().getMessage(), containsString("failed to execute script")); assertThat(bulkResponse.getItems()[0].getResponse(), nullValue()); - assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getId(), equalTo("2")); - assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(2L)); - assertThat(((Integer) ((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getGetResult().field("field").getValue()), - equalTo(2)); + assertThat(bulkResponse.getItems()[1].getResponse().getId(), equalTo("2")); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L)); + assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getGetResult().field("field").getValue(), equalTo(2)); assertThat(bulkResponse.getItems()[1].getFailure(), nullValue()); assertThat(bulkResponse.getItems()[2].getFailure().getId(), equalTo("3")); @@ -248,17 +299,19 @@ public class BulkTests extends ESIntegTestCase { if (numDocs % 2 == 1) { numDocs++; // this test needs an even num of docs } - logger.info("Bulk-Indexing {} docs", numDocs); + + final Script script = new Script("ctx._source.counter += 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + BulkRequestBuilder builder = client().prepareBulk(); for (int i = 0; i < numDocs; i++) { builder.add( client().prepareUpdate() - .setIndex("test").setType("type1").setId(Integer.toString(i)) - .setScript(new Script("ctx._source.counter += 1", ScriptService.ScriptType.INLINE, null, null)).setFields("counter") - .setUpsert(jsonBuilder().startObject().field("counter", 1).endObject())); + .setIndex("test").setType("type1").setId(Integer.toString(i)).setFields("counter") + .setScript(script) + .setUpsert(jsonBuilder().startObject().field("counter", 1).endObject())); } - BulkResponse response = builder.execute().actionGet(); + BulkResponse response = builder.get(); assertThat(response.hasFailures(), equalTo(false)); assertThat(response.getItems().length, equalTo(numDocs)); for (int i = 0; i < numDocs; i++) { @@ -267,10 +320,9 @@ public class BulkTests extends ESIntegTestCase { assertThat(response.getItems()[i].getIndex(), equalTo("test")); assertThat(response.getItems()[i].getType(), equalTo("type1")); assertThat(response.getItems()[i].getOpType(), equalTo("update")); - assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getId(), equalTo(Integer.toString(i))); - assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getVersion(), equalTo(1L)); - assertThat(((Integer) ((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue()), - equalTo(1)); + assertThat(response.getItems()[i].getResponse().getId(), equalTo(Integer.toString(i))); + assertThat(response.getItems()[i].getResponse().getVersion(), equalTo(1L)); + assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue(), equalTo(1)); for (int j = 0; j < 5; j++) { GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).setFields("counter").execute() @@ -286,7 +338,7 @@ public class BulkTests extends ESIntegTestCase { UpdateRequestBuilder updateBuilder = client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i)) .setFields("counter"); if (i % 2 == 0) { - updateBuilder.setScript(new Script("ctx._source.counter += 1", ScriptService.ScriptType.INLINE, null, null)); + updateBuilder.setScript(script); } else { updateBuilder.setDoc(jsonBuilder().startObject().field("counter", 2).endObject()); } @@ -306,17 +358,15 @@ public class BulkTests extends ESIntegTestCase { assertThat(response.getItems()[i].getIndex(), equalTo("test")); assertThat(response.getItems()[i].getType(), equalTo("type1")); assertThat(response.getItems()[i].getOpType(), equalTo("update")); - assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getId(), equalTo(Integer.toString(i))); - assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getVersion(), equalTo(2L)); - assertThat(((Integer) ((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue()), - equalTo(2)); + assertThat(response.getItems()[i].getResponse().getId(), equalTo(Integer.toString(i))); + assertThat(response.getItems()[i].getResponse().getVersion(), equalTo(2L)); + assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue(), equalTo(2)); } builder = client().prepareBulk(); int maxDocs = numDocs / 2 + numDocs; for (int i = (numDocs / 2); i < maxDocs; i++) { - builder.add(client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i)) - .setScript(new Script("ctx._source.counter += 1", ScriptService.ScriptType.INLINE, null, null))); + builder.add(client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i)).setScript(script)); } response = builder.execute().actionGet(); assertThat(response.hasFailures(), equalTo(true)); @@ -338,7 +388,7 @@ public class BulkTests extends ESIntegTestCase { builder = client().prepareBulk(); for (int i = 0; i < numDocs; i++) { builder.add(client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i)) - .setScript(new Script("ctx.op = \"none\"", ScriptService.ScriptType.INLINE, null, null))); + .setScript(new Script("ctx.op = \"none\"", ScriptType.INLINE, CustomScriptPlugin.NAME, null))); } response = builder.execute().actionGet(); assertThat(response.buildFailureMessage(), response.hasFailures(), equalTo(false)); @@ -354,7 +404,7 @@ public class BulkTests extends ESIntegTestCase { builder = client().prepareBulk(); for (int i = 0; i < numDocs; i++) { builder.add(client().prepareUpdate().setIndex("test").setType("type1").setId(Integer.toString(i)) - .setScript(new Script("ctx.op = \"delete\"", ScriptService.ScriptType.INLINE, null, null))); + .setScript(new Script("ctx.op = \"delete\"", ScriptType.INLINE, CustomScriptPlugin.NAME, null))); } response = builder.execute().actionGet(); assertThat(response.hasFailures(), equalTo(false)); @@ -416,11 +466,38 @@ public class BulkTests extends ESIntegTestCase { BulkRequestBuilder builder = client().prepareBulk(); - byte[] addParent = new BytesArray("{\"index\" : { \"_index\" : \"test\", \"_type\" : \"parent\", \"_id\" : \"parent1\"}}\n" + - "{\"field1\" : \"value1\"}\n").array(); + // It's important to use JSON parsing here and request objects: issue 3444 is related to incomplete option parsing + byte[] addParent = new BytesArray( + "{" + + " \"index\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"parent\"," + + " \"_id\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"field1\" : \"value1\"" + + "}" + + "\n").array(); - byte[] addChild = new BytesArray("{ \"update\" : { \"_index\" : \"test\", \"_type\" : \"child\", \"_id\" : \"child1\", \"parent\" : \"parent1\"}}\n" + - "{\"doc\" : { \"field1\" : \"value1\"}, \"doc_as_upsert\" : \"true\"}\n").array(); + byte[] addChild = new BytesArray( + "{" + + " \"update\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"child\"," + + " \"_id\" : \"child1\"," + + " \"parent\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"doc\" : {" + + " \"field1\" : \"value1\"" + + " }," + + " \"doc_as_upsert\" : \"true\"" + + "}" + + "\n").array(); builder.add(addParent, 0, addParent.length); builder.add(addChild, 0, addChild.length); @@ -452,14 +529,57 @@ public class BulkTests extends ESIntegTestCase { BulkRequestBuilder builder = client().prepareBulk(); - byte[] addParent = new BytesArray("{\"index\" : { \"_index\" : \"test\", \"_type\" : \"parent\", \"_id\" : \"parent1\"}}\n" + - "{\"field1\" : \"value1\"}\n").array(); + byte[] addParent = new BytesArray( + "{" + + " \"index\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"parent\"," + + " \"_id\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"field1\" : \"value1\"" + + "}" + + "\n").array(); - byte[] addChild1 = new BytesArray("{\"update\" : { \"_id\" : \"child1\", \"_type\" : \"child\", \"_index\" : \"test\", \"parent\" : \"parent1\"} }\n" + - "{ \"script\" : {\"inline\" : \"ctx._source.field2 = 'value2'\"}, \"upsert\" : {\"field1\" : \"value1\"}}\n").array(); + byte[] addChild1 = new BytesArray( + "{" + + " \"update\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"child\"," + + " \"_id\" : \"child1\"," + + " \"parent\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"script\" : {" + + " \"inline\" : \"ctx._source.field2 = 'value2'\"" + + " }," + + " \"upsert\" : {" + + " \"field1\" : \"value1'\"" + + " }" + + "}" + + "\n").array(); - byte[] addChild2 = new BytesArray("{\"update\" : { \"_id\" : \"child1\", \"_type\" : \"child\", \"_index\" : \"test\", \"parent\" : \"parent1\"} }\n" + - "{ \"script\" : \"ctx._source.field2 = 'value2'\", \"upsert\" : {\"field1\" : \"value1\"}}\n").array(); + byte[] addChild2 = new BytesArray( + "{" + + " \"update\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"child\"," + + " \"_id\" : \"child1\"," + + " \"parent\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"script\" : \"ctx._source.field2 = 'value2'\"," + + " \"upsert\" : {" + + " \"field1\" : \"value1'\"" + + " }" + + "}" + + "\n").array(); builder.add(addParent, 0, addParent.length); builder.add(addChild1, 0, addChild1.length); @@ -490,15 +610,48 @@ public class BulkTests extends ESIntegTestCase { BulkRequestBuilder builder = client().prepareBulk(); - byte[] addParent = new BytesArray("{\"index\" : { \"_index\" : \"test\", \"_type\" : \"parent\", \"_id\" : \"parent1\"}}\n" - + "{\"field1\" : \"value1\"}\n").array(); + byte[] addParent = new BytesArray( + "{" + + " \"index\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"parent\"," + + " \"_id\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"field1\" : \"value1\"" + + "}" + + "\n").array(); byte[] addChildOK = new BytesArray( - "{\"index\" : { \"_id\" : \"child1\", \"_type\" : \"child\", \"_index\" : \"test\", \"parent\" : \"parent1\"} }\n" - + "{ \"field1\" : \"value1\"}\n").array(); + "{" + + " \"index\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"child\"," + + " \"_id\" : \"child1\"," + + " \"parent\" : \"parent1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"field1\" : \"value1\"" + + "}" + + "\n").array(); + byte[] addChildMissingRouting = new BytesArray( - "{\"index\" : { \"_id\" : \"child2\", \"_type\" : \"child\", \"_index\" : \"test\"} }\n" + "{ \"field1\" : \"value1\"}\n") - .array(); + "{" + + " \"index\" : {" + + " \"_index\" : \"test\"," + + " \"_type\" : \"child\"," + + " \"_id\" : \"child1\"" + + " }" + + "}" + + "\n" + + "{" + + " \"field1\" : \"value1\"" + + "}" + + "\n").array(); builder.add(addParent, 0, addParent.length); builder.add(addChildOK, 0, addChildOK.length); @@ -523,19 +676,16 @@ public class BulkTests extends ESIntegTestCase { for (int i = 0; i < responses.length; i++) { final int threadID = i; - threads[threadID] = new Thread(new Runnable() { - @Override - public void run() { - try { - cyclicBarrier.await(); - } catch (Exception e) { - return; - } - BulkRequestBuilder requestBuilder = client().prepareBulk(); - requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID)); - responses[threadID] = requestBuilder.get(); - + threads[threadID] = new Thread(() -> { + try { + cyclicBarrier.await(); + } catch (Exception e) { + return; } + BulkRequestBuilder requestBuilder = client().prepareBulk(); + requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID)); + responses[threadID] = requestBuilder.get(); + }); threads[threadID].start(); @@ -613,7 +763,6 @@ public class BulkTests extends ESIntegTestCase { assertThat(bulkItemResponse.getItems()[5].getOpType(), is("delete")); } - private static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } diff --git a/core/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java b/core/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java index ec3b5421e0e..1486df298ba 100644 --- a/core/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java +++ b/core/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.support; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; @@ -44,12 +45,14 @@ public class ActiveShardsObserverIT extends ESIntegTestCase { settingsBuilder.put("index.routing.allocation.exclude._name", exclude); } Settings settings = settingsBuilder.build(); - assertFalse(prepareCreate("test-idx") + final String indexName = "test-idx"; + assertFalse(prepareCreate(indexName) .setSettings(settings) .setWaitForActiveShards(randomBoolean() ? ActiveShardCount.from(1) : ActiveShardCount.ALL) .setTimeout("100ms") .get() .isShardsAcked()); + waitForIndexCreationToComplete(indexName); } public void testCreateIndexNoActiveShardsNoWaiting() throws Exception { @@ -74,22 +77,24 @@ public class ActiveShardsObserverIT extends ESIntegTestCase { final int numReplicas = numDataNodes + randomInt(4); Settings settings = Settings.builder() .put(indexSettings()) - .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 7)) + .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)) .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), numReplicas) .build(); - assertFalse(prepareCreate("test-idx") + final String indexName = "test-idx"; + assertFalse(prepareCreate(indexName) .setSettings(settings) .setWaitForActiveShards(ActiveShardCount.from(randomIntBetween(numDataNodes + 1, numReplicas + 1))) .setTimeout("100ms") .get() .isShardsAcked()); + waitForIndexCreationToComplete(indexName); } public void testCreateIndexEnoughActiveShards() throws Exception { final String indexName = "test-idx"; Settings settings = Settings.builder() .put(indexSettings()) - .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 7)) + .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)) .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), internalCluster().numDataNodes() + randomIntBetween(0, 3)) .build(); ActiveShardCount waitForActiveShards = ActiveShardCount.from(randomIntBetween(0, internalCluster().numDataNodes())); @@ -104,12 +109,17 @@ public class ActiveShardsObserverIT extends ESIntegTestCase { .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)) .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), numReplicas) .build(); - assertFalse(prepareCreate("test-idx1") + final String indexName = "test-idx"; + assertFalse(prepareCreate(indexName) .setSettings(settings) .setWaitForActiveShards(ActiveShardCount.ALL) .setTimeout("100ms") .get() .isShardsAcked()); + waitForIndexCreationToComplete(indexName); + if (client().admin().indices().prepareExists(indexName).get().isExists()) { + client().admin().indices().prepareDelete(indexName).get(); + } // enough data nodes, all shards are active settings = Settings.builder() @@ -117,7 +127,7 @@ public class ActiveShardsObserverIT extends ESIntegTestCase { .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 7)) .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), internalCluster().numDataNodes() - 1) .build(); - assertAcked(prepareCreate("test-idx2").setSettings(settings).setWaitForActiveShards(ActiveShardCount.ALL).get()); + assertAcked(prepareCreate(indexName).setSettings(settings).setWaitForActiveShards(ActiveShardCount.ALL).get()); } public void testCreateIndexStopsWaitingWhenIndexDeleted() throws Exception { @@ -145,4 +155,12 @@ public class ActiveShardsObserverIT extends ESIntegTestCase { assertAcked(responseListener.get()); } + // Its possible that the cluster state update task that includes the create index hasn't processed before we timeout, + // and subsequently the test cleanup process does not delete the index in question because it does not see it, and + // only after the test cleanup does the index creation manifest in the cluster state. To take care of this problem + // and its potential ramifications, we wait here for the index creation cluster state update task to finish + private void waitForIndexCreationToComplete(final String indexName) { + client().admin().cluster().prepareHealth(indexName).setWaitForEvents(Priority.URGENT).get(); + } + } diff --git a/core/src/test/java/org/elasticsearch/common/StringsTests.java b/core/src/test/java/org/elasticsearch/common/StringsTests.java index 406b4160519..1b987d47796 100644 --- a/core/src/test/java/org/elasticsearch/common/StringsTests.java +++ b/core/src/test/java/org/elasticsearch/common/StringsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -73,4 +74,32 @@ public class StringsTests extends ESTestCase { assertThat(toString, containsString("\"ok\":\"here\"")); assertThat(toString, containsString("\"catastrophe\":\"\"")); } + + public void testSplitStringToSet() { + assertEquals(Strings.splitStringByCommaToSet(null), Sets.newHashSet()); + assertEquals(Strings.splitStringByCommaToSet(""), Sets.newHashSet()); + assertEquals(Strings.splitStringByCommaToSet("a,b,c"), Sets.newHashSet("a","b","c")); + assertEquals(Strings.splitStringByCommaToSet("a, b, c"), Sets.newHashSet("a","b","c")); + assertEquals(Strings.splitStringByCommaToSet(" a , b, c "), Sets.newHashSet("a","b","c")); + assertEquals(Strings.splitStringByCommaToSet("aa, bb, cc"), Sets.newHashSet("aa","bb","cc")); + assertEquals(Strings.splitStringByCommaToSet(" a "), Sets.newHashSet("a")); + assertEquals(Strings.splitStringByCommaToSet(" a "), Sets.newHashSet("a")); + assertEquals(Strings.splitStringByCommaToSet(" aa "), Sets.newHashSet("aa")); + assertEquals(Strings.splitStringByCommaToSet(" "), Sets.newHashSet()); + + assertEquals(Strings.splitStringToSet(null, ' '), Sets.newHashSet()); + assertEquals(Strings.splitStringToSet("", ' '), Sets.newHashSet()); + assertEquals(Strings.splitStringToSet("a b c", ' '), Sets.newHashSet("a","b","c")); + assertEquals(Strings.splitStringToSet("a, b, c", ' '), Sets.newHashSet("a,","b,","c")); + assertEquals(Strings.splitStringToSet(" a b c ", ' '), Sets.newHashSet("a","b","c")); + assertEquals(Strings.splitStringToSet(" a b c ", ' '), Sets.newHashSet("a","b","c")); + assertEquals(Strings.splitStringToSet("aa bb cc", ' '), Sets.newHashSet("aa","bb","cc")); + assertEquals(Strings.splitStringToSet(" a ", ' '), Sets.newHashSet("a")); + assertEquals(Strings.splitStringToSet(" a ", ' '), Sets.newHashSet("a")); + assertEquals(Strings.splitStringToSet(" a ", ' '), Sets.newHashSet("a")); + assertEquals(Strings.splitStringToSet("a ", ' '), Sets.newHashSet("a")); + assertEquals(Strings.splitStringToSet(" aa ", ' '), Sets.newHashSet("aa")); + assertEquals(Strings.splitStringToSet("aa ", ' '), Sets.newHashSet("aa")); + assertEquals(Strings.splitStringToSet(" ", ' '), Sets.newHashSet()); + } } diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 50e05d97985..96c52d9dc8e 100644 --- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -74,10 +74,10 @@ public class NodeEnvironmentTests extends ESTestCase { } public void testNodeLockSingleEnvironment() throws IOException { - NodeEnvironment env = newNodeEnvironment(Settings.builder() - .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 1).build()); - Settings settings = env.getSettings(); - List dataPaths = Environment.PATH_DATA_SETTING.get(env.getSettings()); + final Settings settings = buildEnvSettings(Settings.builder() + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 1).build()); + NodeEnvironment env = newNodeEnvironment(settings); + List dataPaths = Environment.PATH_DATA_SETTING.get(settings); try { // Reuse the same location and attempt to lock again @@ -121,9 +121,10 @@ public class NodeEnvironmentTests extends ESTestCase { } public void testNodeLockMultipleEnvironment() throws IOException { - final NodeEnvironment first = newNodeEnvironment(); - List dataPaths = Environment.PATH_DATA_SETTING.get(first.getSettings()); - NodeEnvironment second = new NodeEnvironment(first.getSettings(), new Environment(first.getSettings())); + final Settings settings = buildEnvSettings(Settings.EMPTY); + final NodeEnvironment first = newNodeEnvironment(settings); + List dataPaths = Environment.PATH_DATA_SETTING.get(settings); + NodeEnvironment second = new NodeEnvironment(settings, new Environment(settings)); assertEquals(first.nodeDataPaths().length, dataPaths.size()); assertEquals(second.nodeDataPaths().length, dataPaths.size()); for (int i = 0; i < dataPaths.size(); i++) { @@ -388,21 +389,20 @@ public class NodeEnvironmentTests extends ESTestCase { assertThat("index paths uses the regular template", env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID()))); - env.close(); - NodeEnvironment env2 = newNodeEnvironment(dataPaths, "/tmp", + IndexSettings s3 = new IndexSettings(s2.getIndexMetaData(), Settings.builder().put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), false).build()); - assertThat(env2.availableShardPaths(sid), equalTo(env2.availableShardPaths(sid))); - assertThat(env2.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/" + index.getUUID() + "/0"))); + assertThat(env.availableShardPaths(sid), equalTo(env.availableShardPaths(sid))); + assertThat(env.resolveCustomLocation(s3, sid), equalTo(PathUtils.get("/tmp/foo/" + index.getUUID() + "/0"))); assertThat("shard paths with a custom data_path should contain only regular paths", - env2.availableShardPaths(sid), + env.availableShardPaths(sid), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0"))); assertThat("index paths uses the regular template", - env2.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID()))); + env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID()))); - env2.close(); + env.close(); } public void testWhetherClusterFolderShouldBeUsed() throws Exception { @@ -495,13 +495,17 @@ public class NodeEnvironmentTests extends ESTestCase { @Override public NodeEnvironment newNodeEnvironment(Settings settings) throws IOException { - Settings build = Settings.builder() - .put(settings) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) - .putArray(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()).build(); + Settings build = buildEnvSettings(settings); return new NodeEnvironment(build, new Environment(build)); } + public Settings buildEnvSettings(Settings settings) { + return Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) + .putArray(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) + .put(settings).build(); + } + public NodeEnvironment newNodeEnvironment(String[] dataPaths, Settings settings) throws IOException { Settings build = Settings.builder() .put(settings) diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java index f9de466e978..87167cdb733 100644 --- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java +++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java @@ -189,11 +189,11 @@ public class HttpServerTests extends ESTestCase { } private static final class TestRestRequest extends RestRequest { - private final String path; + private final BytesReference content; private TestRestRequest(String path, String content) { - this.path = path; + super(Collections.emptyMap(), path); this.content = new BytesArray(content); } @@ -207,11 +207,6 @@ public class HttpServerTests extends ESTestCase { return null; } - @Override - public String rawPath() { - return path; - } - @Override public boolean hasContent() { return true; @@ -232,24 +227,5 @@ public class HttpServerTests extends ESTestCase { return null; } - @Override - public boolean hasParam(String key) { - return false; - } - - @Override - public String param(String key) { - return null; - } - - @Override - public String param(String key, String defaultValue) { - return null; - } - - @Override - public Map params() { - return null; - } } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index 749b1621e4d..f3433a9669a 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -140,7 +140,8 @@ public class ShardPathTests extends ESTestCase { Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), path); - ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings)); + ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, + IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings, nodeSettings)); boolean found = false; for (Path p : env.nodeDataPaths()) { if (p.equals(shardPath.getRootStatePath())) { diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index b448f35c21b..361bca6076a 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.After; import org.junit.Before; @@ -59,6 +60,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; @@ -316,9 +318,49 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { client.prepareSearch("cb-test").setQuery(matchAllQuery()).addAggregation(cardinality("card").field("test")).get(); fail("aggregation should have tripped the breaker"); } catch (Exception e) { - String errMsg = "CircuitBreakingException[[request] Data too large, data for [] would be larger than limit of [10/10b]]"; + String errMsg = "CircuitBreakingException[[request] Data too large"; assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg)); + errMsg = "would be larger than limit of [10/10b]]"; + assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg)); + } + } + + public void testBucketBreaker() throws Exception { + if (noopBreakerUsed()) { + logger.info("--> noop breakers used, skipping test"); + return; + } + assertAcked(prepareCreate("cb-test", 1, Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)))); + Client client = client(); + + // Make request breaker limited to a small amount + Settings resetSettings = Settings.builder() + .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "100b") + .build(); + assertAcked(client.admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings)); + + // index some different terms so we have some field data for loading + int docCount = scaledRandomIntBetween(100, 1000); + List reqs = new ArrayList<>(); + for (long id = 0; id < docCount; id++) { + reqs.add(client.prepareIndex("cb-test", "type", Long.toString(id)).setSource("test", id)); + } + indexRandom(true, reqs); + + // A terms aggregation on the "test" field should trip the bucket circuit breaker + try { + SearchResponse resp = client.prepareSearch("cb-test") + .setQuery(matchAllQuery()) + .addAggregation(terms("my_terms").field("test")) + .get(); + assertTrue("there should be shard failures", resp.getFailedShards() > 0); + fail("aggregation should have tripped the breaker"); + } catch (Exception e) { + String errMsg = "CircuitBreakingException[[request] " + + "Data too large, data for [] would be larger than limit of [100/100b]]"; + assertThat("Exception: " + e.toString() + " should contain a CircuitBreakingException", + e.toString(), containsString(errMsg)); } } diff --git a/core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java index 6cb2c9dd53f..be6ec1059d6 100644 --- a/core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java @@ -193,4 +193,17 @@ public class CompoundProcessorTests extends ESTestCase { assertThat(firstProcessor.getInvokedCounter(), equalTo(1)); assertThat(secondProcessor.getInvokedCounter(), equalTo(1)); } + + public void testBreakOnFailure() throws Exception { + TestProcessor firstProcessor = new TestProcessor("id1", "first", ingestDocument -> {throw new RuntimeException("error1");}); + TestProcessor secondProcessor = new TestProcessor("id2", "second", ingestDocument -> {throw new RuntimeException("error2");}); + TestProcessor onFailureProcessor = new TestProcessor("id2", "on_failure", ingestDocument -> {}); + CompoundProcessor pipeline = new CompoundProcessor(false, Arrays.asList(firstProcessor, secondProcessor), + Collections.singletonList(onFailureProcessor)); + pipeline.execute(ingestDocument); + assertThat(firstProcessor.getInvokedCounter(), equalTo(1)); + assertThat(secondProcessor.getInvokedCounter(), equalTo(0)); + assertThat(onFailureProcessor.getInvokedCounter(), equalTo(1)); + + } } diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java index 5637e937c1f..2dc95f8e9f1 100644 --- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java +++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java @@ -54,13 +54,13 @@ public class InternalSettingsPreparerTests extends ESTestCase { public void testEmptySettings() { Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY); - assertNotNull(settings.get("node.name")); // a name was set + assertNull(settings.get("node.name")); // a name was not set assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set int size = settings.names().size(); Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null); settings = env.settings(); - assertNotNull(settings.get("node.name")); // a name was set + assertNull(settings.get("node.name")); // a name was not set assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set assertEquals(settings.toString(), size + 1 /* path.home is in the base settings */, settings.names().size()); String home = Environment.PATH_HOME_SETTING.get(baseEnvSettings); diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 051159b448b..642b3a53a34 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.Index; import org.elasticsearch.rest.support.RestUtils; import org.elasticsearch.search.SearchShardTarget; @@ -33,11 +34,15 @@ import org.elasticsearch.transport.RemoteTransportException; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Collections; +import java.util.Map; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -157,8 +162,37 @@ public class BytesRestResponseTests extends ESTestCase { public void testResponseWhenPathContainsEncodingError() throws IOException { final String path = "%a"; - final RestRequest request = mock(RestRequest.class); - when(request.rawPath()).thenReturn(path); + final RestRequest request = new RestRequest(Collections.emptyMap(), path) { + @Override + public Method method() { + return null; + } + + @Override + public String uri() { + return null; + } + + @Override + public boolean hasContent() { + return false; + } + + @Override + public BytesReference content() { + return null; + } + + @Override + public String header(String name) { + return null; + } + + @Override + public Iterable> headers() { + return null; + } + }; final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath())); final RestChannel channel = new DetailedExceptionRestChannel(request); // if we try to decode the path, this will throw an IllegalArgumentException again diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java index 4a0369a4019..2c320288edf 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java @@ -49,15 +49,10 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin { protected Map, Object>> pluginScripts() { Map, Object>> scripts = new HashMap<>(); - scripts.put("20 - _value", vars -> { - double value = (double) vars.get("_value"); - return 20.0d - value; - }); - - scripts.put("_value - 1", vars -> { - double value = (double) vars.get("_value"); - return value - 1.0d; - }); + scripts.put("20 - _value", vars -> 20.0d - (double) vars.get("_value")); + scripts.put("_value - 1", vars -> (double) vars.get("_value") - 1); + scripts.put("_value + 1", vars -> (double) vars.get("_value") + 1); + scripts.put("_value * -1", vars -> (double) vars.get("_value") * -1); scripts.put("_value - dec", vars -> { double value = (double) vars.get("_value"); @@ -65,6 +60,12 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin { return value - dec; }); + scripts.put("_value + inc", vars -> { + double value = (double) vars.get("_value"); + int inc = (int) vars.get("inc"); + return value + inc; + }); + scripts.put("doc['value'].value", vars -> { Map doc = (Map) vars.get("doc"); return doc.get("value"); @@ -77,6 +78,13 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin { return value.getValue() - dec; }); + scripts.put("doc['value'].value + inc", vars -> { + int inc = (int) vars.get("inc"); + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Longs value = (ScriptDocValues.Longs) doc.get("value"); + return value.getValue() + inc; + }); + scripts.put("doc['values'].values", vars -> { Map doc = (Map) vars.get("doc"); return doc.get("values"); @@ -94,7 +102,17 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin { return res; }); - scripts.put("_value * -1", vars -> (double) vars.get("_value") * -1); + scripts.put("[ doc['value'].value, doc['value'].value - dec ]", vars -> { + Long a = ((ScriptDocValues.Longs) scripts.get("doc['value'].value").apply(vars)).getValue(); + Long b = (Long) scripts.get("doc['value'].value - dec").apply(vars); + return new Long[]{a, b}; + }); + + scripts.put("[ doc['value'].value, doc['value'].value + inc ]", vars -> { + Long a = ((ScriptDocValues.Longs) scripts.get("doc['value'].value").apply(vars)).getValue(); + Long b = (Long) scripts.get("doc['value'].value + inc").apply(vars); + return new Long[]{a, b}; + }); return scripts; } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 2c3534183e0..ace55c2c616 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -380,6 +380,7 @@ public class DateHistogramIT extends ESIntegTestCase { assertThat(histo.getName(), equalTo("histo")); List buckets = histo.getBuckets(); assertThat(buckets.size(), equalTo(3)); + assertThat(histo.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) histo.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count"); Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value"); @@ -600,7 +601,7 @@ public class DateHistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); Histogram.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java index 36613cfa784..1e250681004 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java @@ -137,7 +137,7 @@ public class DateRangeIT extends ESIntegTestCase { assertThat(range.getBuckets().size(), equalTo(3)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(range.getBuckets()); + List buckets = new ArrayList<>(range.getBuckets()); Range.Bucket bucket = buckets.get(0); assertThat((String) bucket.getKey(), equalTo("a long time ago")); @@ -421,6 +421,7 @@ public class DateRangeIT extends ESIntegTestCase { assertThat(range.getName(), equalTo("range")); List buckets = range.getBuckets(); assertThat(buckets.size(), equalTo(3)); + assertThat(range.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) range.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) range.getProperty("_count"); Object[] propertiesCounts = (Object[]) range.getProperty("sum.value"); @@ -855,7 +856,7 @@ public class DateRangeIT extends ESIntegTestCase { Range dateRange = bucket.getAggregations().get("date_range"); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(dateRange.getBuckets()); + List buckets = new ArrayList<>(dateRange.getBuckets()); assertThat(dateRange, Matchers.notNullValue()); assertThat(dateRange.getName(), equalTo("date_range")); assertThat(buckets.size(), is(1)); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java similarity index 92% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java index adbc66c2202..ed43ba8bd63 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java @@ -16,18 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScoreAccessor; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -50,10 +50,13 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -69,15 +72,47 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase -public class DoubleTermsTests extends AbstractTermsTestCase { +public class DoubleTermsIT extends AbstractTermsTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends AggregationTestScriptsPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = super.pluginScripts(); + + scripts.put("(long) (_value / 1000 + 1)", vars -> (long) ((double) vars.get("_value") / 1000 + 1)); + + scripts.put("doc['" + MULTI_VALUED_FIELD_NAME + "']", vars -> { + Map doc = (Map) vars.get("doc"); + return doc.get(MULTI_VALUED_FIELD_NAME); + }); + + scripts.put("doc['" + MULTI_VALUED_FIELD_NAME + "'].value", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles value = (ScriptDocValues.Doubles) doc.get(MULTI_VALUED_FIELD_NAME); + return value.getValue(); + }); + + scripts.put("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles value = (ScriptDocValues.Doubles) doc.get(SINGLE_VALUED_FIELD_NAME); + return value.getValue(); + }); + + scripts.put("ceil(_score.doubleValue()/3)", vars -> { + ScoreAccessor score = (ScoreAccessor) vars.get("_score"); + return Math.ceil(score.doubleValue() / 3); + }); + + return scripts; + } } private static final int NUM_DOCS = 5; // TODO: randomize the size? @@ -391,6 +426,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { assertThat(terms, notNullValue()); assertThat(terms.getName(), equalTo("terms")); assertThat(terms.getBuckets().size(), equalTo(5)); + assertThat(terms.getProperty("_bucket_count"), equalTo(5)); Object[] propertiesKeys = (Object[]) terms.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) terms.getProperty("_count"); Object[] propertiesCounts = (Object[]) terms.getProperty("sum.value"); @@ -415,8 +451,8 @@ public class DoubleTermsTests extends AbstractTermsTestCase { .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) - .script(new Script("_value + 1"))) - .execute().actionGet(); + .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -468,8 +504,8 @@ public class DoubleTermsTests extends AbstractTermsTestCase { .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) - .script(new Script("_value + 1"))) - .execute().actionGet(); + .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -497,8 +533,8 @@ public class DoubleTermsTests extends AbstractTermsTestCase { .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) - .script(new Script("(long) (_value / 1000 + 1)"))) - .execute().actionGet(); + .script(new Script("(long) (_value / 1000 + 1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -537,8 +573,11 @@ public class DoubleTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).script( - new Script("doc['" + MULTI_VALUED_FIELD_NAME + "'].value"))).execute().actionGet(); + terms("terms") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, + CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -561,8 +600,11 @@ public class DoubleTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).script( - new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']"))).execute().actionGet(); + terms("terms") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, + CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -1035,14 +1077,20 @@ public class DoubleTermsTests extends AbstractTermsTestCase { } public void testScriptScore() { + Script scoringScript = + new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin .NAME, null); + + Script aggregationScript = new Script("ceil(_score.doubleValue()/3)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setTypes("type") - .setQuery( - functionScoreQuery(ScoreFunctionBuilders.scriptFunction(new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")))) + .setQuery(functionScoreQuery(scriptFunction(scoringScript))) .addAggregation( - terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).script( - new Script("ceil(_score.doubleValue()/3)"))).execute().actionGet(); + terms("terms") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .script(aggregationScript)) + .get(); assertSearchResponse(response); @@ -1087,7 +1135,9 @@ public class DoubleTermsTests extends AbstractTermsTestCase { public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception { double[] expectedKeys = new double[] { 2, 1, 4, 5, 3, 6, 7 }; - assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false)); + assertMultiSortResponse(expectedKeys, Terms.Order.count(false), + Terms.Order.aggregation("sum_d", false), + Terms.Order.aggregation("avg_l", false)); } public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java index 592861ccce2..619bba7da12 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java @@ -172,6 +172,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(filters.getName(), equalTo("tags")); assertThat(filters.getBuckets().size(), equalTo(2)); + assertThat(filters.getProperty("_bucket_count"), equalTo(2)); Object[] propertiesKeys = (Object[]) filters.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) filters.getProperty("_count"); Object[] propertiesCounts = (Object[]) filters.getProperty("avg_value.value"); @@ -426,6 +427,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(filters.getName(), equalTo("tags")); assertThat(filters.getBuckets().size(), equalTo(3)); + assertThat(filters.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) filters.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) filters.getProperty("_count"); Object[] propertiesCounts = (Object[]) filters.getProperty("avg_value.value"); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index a27b193f435..a69513af77d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -349,6 +349,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(geoDist.getName(), equalTo("amsterdam_rings")); List buckets = geoDist.getBuckets(); assertThat(geoDist.getBuckets().size(), equalTo(3)); + assertThat(geoDist.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) geoDist.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) geoDist.getProperty("_count"); Object[] propertiesCities = (Object[]) geoDist.getProperty("cities"); @@ -429,7 +430,7 @@ public class GeoDistanceIT extends ESIntegTestCase { Range geoDistance = bucket.getAggregations().get("geo_dist"); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(geoDistance.getBuckets()); + List buckets = new ArrayList<>(geoDistance.getBuckets()); assertThat(geoDistance, Matchers.notNullValue()); assertThat(geoDistance.getName(), equalTo("geo_dist")); assertThat(buckets.size(), is(1)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java index dc439318978..8df4a8ca608 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java @@ -26,7 +26,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -253,7 +252,7 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -276,7 +275,7 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(numValueBuckets - i - 1); assertThat(bucket, notNullValue()); @@ -300,7 +299,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet buckets = new LongHashSet(); // TODO: use diamond once JI-9019884 is fixed - List histoBuckets = new ArrayList(histo.getBuckets()); + List histoBuckets = new ArrayList<>(histo.getBuckets()); long previousCount = Long.MIN_VALUE; for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = histoBuckets.get(i); @@ -329,7 +328,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet buckets = new LongHashSet(); // TODO: use diamond once JI-9019884 is fixed - List histoBuckets = new ArrayList(histo.getBuckets()); + List histoBuckets = new ArrayList<>(histo.getBuckets()); long previousCount = Long.MAX_VALUE; for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = histoBuckets.get(i); @@ -356,12 +355,13 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo, notNullValue()); assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); + assertThat(histo.getProperty("_bucket_count"), equalTo(numValueBuckets)); Object[] propertiesKeys = (Object[]) histo.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count"); Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value"); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -404,7 +404,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double previousSum = Double.NEGATIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -448,7 +448,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double previousSum = Double.POSITIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -492,7 +492,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double previousSum = Double.POSITIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -538,7 +538,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double prevMax = asc ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -625,7 +625,7 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(numValuesBuckets)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValuesBuckets; ++i) { Histogram.Bucket bucket = buckets.get(numValuesBuckets - i - 1); assertThat(bucket, notNullValue()); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java similarity index 94% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java index 7bb1ed6fd65..6d5e190b542 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java @@ -16,17 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -49,9 +49,11 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -67,15 +69,36 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase -public class LongTermsTests extends AbstractTermsTestCase { +public class LongTermsIT extends AbstractTermsTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends AggregationTestScriptsPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = super.pluginScripts(); + + scripts.put("floor(_value / 1000 + 1)", vars -> Math.floor((double) vars.get("_value") / 1000 + 1)); + + scripts.put("doc['" + MULTI_VALUED_FIELD_NAME + "']", vars -> { + Map doc = (Map) vars.get("doc"); + return doc.get(MULTI_VALUED_FIELD_NAME); + }); + + scripts.put("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Longs value = (ScriptDocValues.Longs) doc.get(SINGLE_VALUED_FIELD_NAME); + return value.getValue(); + }); + + return scripts; + } } private static final int NUM_DOCS = 5; // TODO randomize the size? @@ -418,8 +441,8 @@ public class LongTermsTests extends AbstractTermsTestCase { .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) - .script(new Script("_value + 1"))) - .execute().actionGet(); + .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -471,8 +494,8 @@ public class LongTermsTests extends AbstractTermsTestCase { .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) - .script(new Script("_value - 1"))) - .execute().actionGet(); + .script(new Script("_value - 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -500,8 +523,8 @@ public class LongTermsTests extends AbstractTermsTestCase { .addAggregation(terms("terms") .field(MULTI_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) - .script(new Script("floor(_value / 1000 + 1)"))) - .execute().actionGet(); + .script(new Script("floor(_value / 1000 + 1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -536,12 +559,13 @@ public class LongTermsTests extends AbstractTermsTestCase { */ public void testScriptSingleValue() throws Exception { + Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) - .script( - new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value"))) - .execute().actionGet(); + .script(script)) + .get(); assertSearchResponse(response); @@ -561,12 +585,13 @@ public class LongTermsTests extends AbstractTermsTestCase { } public void testScriptMultiValued() throws Exception { + Script script = new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation(terms("terms") .collectMode(randomFrom(SubAggCollectionMode.values())) - .script( - new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']"))) - .execute().actionGet(); + .script(script)) + .get(); assertSearchResponse(response); @@ -711,9 +736,11 @@ public class LongTermsTests extends AbstractTermsTestCase { .field(SINGLE_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) .order(Terms.Order.aggregation("avg_i", asc)) - .subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME)).subAggregation(terms("subTerms").field(MULTI_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - ).execute().actionGet(); + .subAggregation( + avg("avg_i").field(SINGLE_VALUED_FIELD_NAME)) + .subAggregation( + terms("subTerms").field(MULTI_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))) + ).get(); assertSearchResponse(response); @@ -753,8 +780,8 @@ public class LongTermsTests extends AbstractTermsTestCase { .field("num_tag") .collectMode(randomFrom(SubAggCollectionMode.values())) .order(Terms.Order.aggregation("filter", asc)) -.subAggregation(filter("filter", QueryBuilders.matchAllQuery())) - ).execute().actionGet(); + .subAggregation(filter("filter", QueryBuilders.matchAllQuery())) + ).get(); assertSearchResponse(response); @@ -1064,7 +1091,9 @@ public class LongTermsTests extends AbstractTermsTestCase { public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception { long[] expectedKeys = new long[] { 2, 1, 4, 5, 3, 6, 7 }; - assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false)); + assertMultiSortResponse(expectedKeys, Terms.Order.count(false), + Terms.Order.aggregation("sum_d", false), + Terms.Order.aggregation("avg_l", false)); } public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception { diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java similarity index 85% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java index 640c00b291d..012df7bfbff 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; @@ -25,12 +25,13 @@ import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -44,10 +45,13 @@ import org.joda.time.format.DateTimeFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -58,18 +62,45 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; - @ESIntegTestCase.SuiteScopeTestCase -public class MinDocCountTests extends AbstractTermsTestCase { +public class MinDocCountIT extends AbstractTermsTestCase { + + private static final QueryBuilder QUERY = QueryBuilders.termQuery("match", true); + private static int cardinality; @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); } - private static final QueryBuilder QUERY = QueryBuilders.termQuery("match", true); + public static class CustomScriptPlugin extends AggregationTestScriptsPlugin { - private static int cardinality; + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + + scripts.put("doc['d'].values", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles value = (ScriptDocValues.Doubles) doc.get("d"); + return value.getValues(); + }); + + scripts.put("doc['l'].values", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Longs value = (ScriptDocValues.Longs) doc.get("l"); + return value.getValues(); + }); + + scripts.put("doc['s'].values", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Strings value = (ScriptDocValues.Strings) doc.get("s"); + return value.getValues(); + }); + + return scripts; + } + } @Override public void setupSuiteScopeCluster() throws Exception { @@ -90,7 +121,8 @@ public class MinDocCountTests extends AbstractTermsTestCase { longTerm = randomInt(cardinality * 2); } while (!longTerms.add(longTerm)); double doubleTerm = longTerm * Math.PI; - String dateTerm = DateTimeFormat.forPattern("yyyy-MM-dd").print(new DateTime(2014, 1, ((int) longTerm % 20) + 1, 0, 0, DateTimeZone.UTC)); + String dateTerm = DateTimeFormat.forPattern("yyyy-MM-dd") + .print(new DateTime(2014, 1, ((int) longTerm % 20) + 1, 0, 0, DateTimeZone.UTC)); final int frequency = randomBoolean() ? 1 : randomIntBetween(2, 20); for (int j = 0; j < frequency; ++j) { indexRequests.add(client().prepareIndex("idx", "type").setSource(jsonBuilder() @@ -119,7 +151,8 @@ public class MinDocCountTests extends AbstractTermsTestCase { YES { @Override TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field) { - return builder.script(new org.elasticsearch.script.Script("doc['" + field + "'].values")); + return builder.script(new org.elasticsearch.script.Script("doc['" + field + "'].values", ScriptService.ScriptType.INLINE, + CustomScriptPlugin.NAME, null)); } }; abstract TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field); @@ -272,7 +305,7 @@ public class MinDocCountTests extends AbstractTermsTestCase { testMinDocCountOnTerms(field, script, order, null, true); } - private void testMinDocCountOnTerms(String field, Script script, Terms.Order order, String include, boolean retryOnFailure) throws Exception { + private void testMinDocCountOnTerms(String field, Script script, Terms.Order order, String include, boolean retry) throws Exception { // all terms final SearchResponse allTermsResponse = client().prepareSearch("idx").setTypes("type") .setSize(0) @@ -307,7 +340,7 @@ public class MinDocCountTests extends AbstractTermsTestCase { assertAllSuccessful(response); assertSubset(allTerms, (Terms) response.getAggregations().get("terms"), minDocCount, size, include); } catch (AssertionError ae) { - if (!retryOnFailure) { + if (!retry) { throw ae; } logger.info("test failed. trying to see if it recovers after 1m.", ae); @@ -380,8 +413,13 @@ public class MinDocCountTests extends AbstractTermsTestCase { final SearchResponse allResponse = client().prepareSearch("idx").setTypes("type") .setSize(0) .setQuery(QUERY) - .addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).order(order).minDocCount(0)) - .execute().actionGet(); + .addAggregation( + dateHistogram("histo") + .field("date") + .dateHistogramInterval(DateHistogramInterval.DAY) + .order(order) + .minDocCount(0)) + .get(); final Histogram allHisto = allResponse.getAggregations().get("histo"); @@ -389,9 +427,14 @@ public class MinDocCountTests extends AbstractTermsTestCase { final SearchResponse response = client().prepareSearch("idx").setTypes("type") .setSize(0) .setQuery(QUERY) - .addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).order(order).minDocCount(minDocCount)) - .execute().actionGet(); - assertSubset(allHisto, (Histogram) response.getAggregations().get("histo"), minDocCount); + .addAggregation( + dateHistogram("histo") + .field("date") + .dateHistogramInterval(DateHistogramInterval.DAY) + .order(order) + .minDocCount(minDocCount)) + .get(); + assertSubset(allHisto, response.getAggregations().get("histo"), minDocCount); } } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java index b3241fc608c..c8b7aa6ad55 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; @@ -44,6 +45,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; @@ -62,15 +64,12 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.core.IsNull.notNullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase public class NestedIT extends ESIntegTestCase { - static int numParents; - static int[] numChildren; - static SubAggCollectionMode aggCollectionMode; + private static int numParents; + private static int[] numChildren; + private static SubAggCollectionMode aggCollectionMode; @Override public void setupSuiteScopeCluster() throws Exception { @@ -245,7 +244,7 @@ public class NestedIT extends ESIntegTestCase { assertThat(nested, notNullValue()); assertThat(nested.getName(), equalTo("nested")); assertThat(nested.getDocCount(), equalTo(docCount)); - assertThat((long) nested.getProperty("_count"), equalTo(docCount)); + assertThat(nested.getProperty("_count"), equalTo(docCount)); assertThat(nested.getAggregations().asList().isEmpty(), is(false)); LongTerms values = nested.getAggregations().get("values"); @@ -263,7 +262,7 @@ public class NestedIT extends ESIntegTestCase { assertEquals(counts[i], bucket.getDocCount()); } } - assertThat((LongTerms) nested.getProperty("values"), sameInstance(values)); + assertThat(nested.getProperty("values"), sameInstance(values)); } public void testNestedAsSubAggregation() throws Exception { @@ -544,4 +543,126 @@ public class NestedIT extends ESIntegTestCase { assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L)); assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L)); } + + public void testFilterAggInsideNestedAgg() throws Exception { + assertAcked(prepareCreate("classes") + .addMapping("class", jsonBuilder().startObject().startObject("class").startObject("properties") + .startObject("name").field("type", "text").endObject() + .startObject("methods") + .field("type", "nested") + .startObject("properties") + .startObject("name").field("type", "text").endObject() + .startObject("return_type").field("type", "keyword").endObject() + .startObject("parameters") + .field("type", "nested") + .startObject("properties") + .startObject("name").field("type", "text").endObject() + .startObject("type").field("type", "keyword").endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().endObject().endObject())); + + client().prepareIndex("classes", "class", "1").setSource(jsonBuilder().startObject() + .field("name", "QueryBuilder") + .startArray("methods") + .startObject() + .field("name", "toQuery") + .field("return_type", "Query") + .startArray("parameters") + .startObject() + .field("name", "context") + .field("type", "QueryShardContext") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "queryName") + .field("return_type", "QueryBuilder") + .startArray("parameters") + .startObject() + .field("name", "queryName") + .field("type", "String") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "boost") + .field("return_type", "QueryBuilder") + .startArray("parameters") + .startObject() + .field("name", "boost") + .field("type", "float") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject()).get(); + client().prepareIndex("classes", "class", "2").setSource(jsonBuilder().startObject() + .field("name", "Document") + .startArray("methods") + .startObject() + .field("name", "add") + .field("return_type", "void") + .startArray("parameters") + .startObject() + .field("name", "field") + .field("type", "IndexableField") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "removeField") + .field("return_type", "void") + .startArray("parameters") + .startObject() + .field("name", "name") + .field("type", "String") + .endObject() + .endArray() + .endObject() + .startObject() + .field("name", "removeFields") + .field("return_type", "void") + .startArray("parameters") + .startObject() + .field("name", "name") + .field("type", "String") + .endObject() + .endArray() + .endObject() + .endArray() + .endObject()).get(); + refresh(); + + SearchResponse response = client().prepareSearch("classes").addAggregation(nested("to_method", "methods") + .subAggregation(filter("num_string_params", + nestedQuery("methods.parameters", termQuery("methods.parameters.type", "String"), ScoreMode.None))) + ).get(); + Nested toMethods = response.getAggregations().get("to_method"); + Filter numStringParams = toMethods.getAggregations().get("num_string_params"); + assertThat(numStringParams.getDocCount(), equalTo(3L)); + + response = client().prepareSearch("classes").addAggregation(nested("to_method", "methods") + .subAggregation(terms("return_type").field("methods.return_type").subAggregation( + filter("num_string_params", nestedQuery("methods.parameters", termQuery("methods.parameters.type", "String"), ScoreMode.None)) + ) + )).get(); + toMethods = response.getAggregations().get("to_method"); + Terms terms = toMethods.getAggregations().get("return_type"); + Bucket bucket = terms.getBucketByKey("void"); + assertThat(bucket.getDocCount(), equalTo(3L)); + numStringParams = bucket.getAggregations().get("num_string_params"); + assertThat(numStringParams.getDocCount(), equalTo(2L)); + + bucket = terms.getBucketByKey("QueryBuilder"); + assertThat(bucket.getDocCount(), equalTo(2L)); + numStringParams = bucket.getAggregations().get("num_string_params"); + assertThat(numStringParams.getDocCount(), equalTo(1L)); + + bucket = terms.getBucketByKey("Query"); + assertThat(bucket.getDocCount(), equalTo(1L)); + numStringParams = bucket.getAggregations().get("num_string_params"); + assertThat(numStringParams.getDocCount(), equalTo(0L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java index 1d74c03048a..db7c301883a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java @@ -55,9 +55,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.core.IsNull.notNullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase public class ReverseNestedIT extends ESIntegTestCase { @@ -170,9 +167,9 @@ public class ReverseNestedIT extends ESIntegTestCase { assertThat(bucket.getKeyAsString(), equalTo("1")); assertThat(bucket.getDocCount(), equalTo(6L)); ReverseNested reverseNested = bucket.getAggregations().get("nested1_to_field1"); - assertThat((long) reverseNested.getProperty("_count"), equalTo(5L)); + assertThat(reverseNested.getProperty("_count"), equalTo(5L)); Terms tags = reverseNested.getAggregations().get("field1"); - assertThat((Terms) reverseNested.getProperty("field1"), sameInstance(tags)); + assertThat(reverseNested.getProperty("field1"), sameInstance(tags)); List tagsBuckets = new ArrayList<>(tags.getBuckets()); assertThat(tagsBuckets.size(), equalTo(6)); assertThat(tagsBuckets.get(0).getKeyAsString(), equalTo("c")); @@ -472,14 +469,25 @@ public class ReverseNestedIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation(nested("nested2", "nested1.nested2").subAggregation(reverseNested("incorrect").path("nested3"))) - .execute().actionGet(); + .get(); Nested nested = searchResponse.getAggregations().get("nested2"); - assertThat(nested, Matchers.notNullValue()); + assertThat(nested, notNullValue()); assertThat(nested.getName(), equalTo("nested2")); ReverseNested reverseNested = nested.getAggregations().get("incorrect"); assertThat(reverseNested.getDocCount(), is(0L)); + + // Test that parsing the reverse_nested agg doesn't fail, because the parent nested agg is unmapped: + searchResponse = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(nested("incorrect1", "incorrect1").subAggregation(reverseNested("incorrect2").path("incorrect2"))) + .get(); + + nested = searchResponse.getAggregations().get("incorrect1"); + assertThat(nested, notNullValue()); + assertThat(nested.getName(), equalTo("incorrect1")); + assertThat(nested.getDocCount(), is(0L)); } public void testSameParentDocHavingMultipleBuckets() throws Exception { diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java similarity index 93% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java index b098db154da..ba3b9380504 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.ElasticsearchException; @@ -24,14 +24,14 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -56,10 +56,12 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.count; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; @@ -76,18 +78,41 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.hamcrest.core.IsNull.nullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase -public class StringTermsTests extends AbstractTermsTestCase { +public class StringTermsIT extends AbstractTermsTestCase { + private static final String SINGLE_VALUED_FIELD_NAME = "s_value"; private static final String MULTI_VALUED_FIELD_NAME = "s_values"; private static Map> expectedMultiSortBuckets; @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends AggregationTestScriptsPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = super.pluginScripts(); + + scripts.put("'foo_' + _value", vars -> "foo_" + (String) vars.get("_value")); + scripts.put("_value.substring(0,3)", vars -> ((String) vars.get("_value")).substring(0, 3)); + + scripts.put("doc['" + MULTI_VALUED_FIELD_NAME + "']", vars -> { + Map doc = (Map) vars.get("doc"); + return doc.get(MULTI_VALUED_FIELD_NAME); + }); + + scripts.put("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Strings value = (ScriptDocValues.Strings) doc.get(SINGLE_VALUED_FIELD_NAME); + return value.getValue(); + }); + + return scripts; + } } @Override @@ -435,8 +460,11 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("high_card_type") .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME).size(20) - .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.term(true))) // we need to sort by terms cause we're checking the first 20 values + terms("terms") + .executionHint(randomExecutionHint()) + .field(SINGLE_VALUED_FIELD_NAME).size(20) + .collectMode(randomFrom(SubAggCollectionMode.values())) + .order(Terms.Order.term(true))) // we need to sort by terms cause we're checking the first 20 values .execute().actionGet(); assertSearchResponse(response); @@ -542,9 +570,12 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).script(new Script("'foo_' + _value"))).execute() - .actionGet(); + terms("terms") + .executionHint(randomExecutionHint()) + .field(SINGLE_VALUED_FIELD_NAME) + .collectMode(randomFrom(SubAggCollectionMode.values())) + .script(new Script("'foo_' + _value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -566,9 +597,12 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(MULTI_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).script(new Script("_value.substring(0,3)"))) - .execute().actionGet(); + terms("terms") + .executionHint(randomExecutionHint()) + .field(MULTI_VALUED_FIELD_NAME) + .collectMode(randomFrom(SubAggCollectionMode.values())) + .script(new Script("_value.substring(0,3)", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -615,8 +649,11 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").executionHint(randomExecutionHint()).script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']")) - .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet(); + terms("terms") + .executionHint(randomExecutionHint()) + .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .collectMode(randomFrom(SubAggCollectionMode.values()))) + .get(); assertSearchResponse(response); @@ -642,9 +679,12 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(MULTI_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).script(new Script("'foo_' + _value"))).execute() - .actionGet(); + terms("terms") + .executionHint(randomExecutionHint()) + .field(MULTI_VALUED_FIELD_NAME) + .collectMode(randomFrom(SubAggCollectionMode.values())) + .script(new Script("'foo_' + _value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -678,12 +718,17 @@ public class StringTermsTests extends AbstractTermsTestCase { */ public void testScriptSingleValue() throws Exception { + Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).executionHint(randomExecutionHint()) - .script(new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value"))).execute().actionGet(); + terms("terms") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .executionHint(randomExecutionHint()) + .script(script)) + .get(); assertSearchResponse(response); @@ -701,12 +746,17 @@ public class StringTermsTests extends AbstractTermsTestCase { } public void testScriptSingleValueExplicitSingleValue() throws Exception { + Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).executionHint(randomExecutionHint()) - .script(new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value"))).execute().actionGet(); + terms("terms") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .executionHint(randomExecutionHint()) + .script(script)) + .get(); assertSearchResponse(response); @@ -728,8 +778,11 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx") .setTypes("type") .addAggregation( - terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).executionHint(randomExecutionHint()) - .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']"))).execute().actionGet(); + terms("terms") + .collectMode(randomFrom(SubAggCollectionMode.values())) + .executionHint(randomExecutionHint()) + .script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); @@ -821,7 +874,11 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L).minDocCount(0).subAggregation(terms("terms").field("value"))) + histogram("histo") + .field(SINGLE_VALUED_FIELD_NAME) + .interval(1L) + .minDocCount(0) + .subAggregation(terms("terms").field("value"))) .execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); @@ -1143,7 +1200,8 @@ public class StringTermsTests extends AbstractTermsTestCase { .subAggregation(terms("values").field("i").collectMode(randomFrom(SubAggCollectionMode.values())))) .execute().actionGet(); - fail("Expected search to fail when trying to sort terms aggregation by sug-aggregation which is not of a metrics or single-bucket type"); + fail("Expected search to fail when trying to sort terms aggregation by sug-aggregation " + + "which is not of a metrics or single-bucket type"); } catch (ElasticsearchException e) { // expected diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java similarity index 92% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java index d7fc77c9ccb..6f4f891ea65 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java @@ -16,20 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats.Bounds; @@ -53,13 +52,11 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -/** - * - */ -public class ExtendedStatsTests extends AbstractNumericTestCase { +public class ExtendedStatsIT extends AbstractNumericTestCase { + @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(AggregationTestScriptsPlugin.class); } private static double stdDev(int... vals) { @@ -298,7 +295,11 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(extendedStats("stats").field("value").script(new Script("_value + 1")).sigma(sigma)) + .addAggregation( + extendedStats("stats") + .field("value") + .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null)) + .sigma(sigma)) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -325,7 +326,9 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - extendedStats("stats").field("value").script(new Script("_value + inc", ScriptType.INLINE, null, params)) + extendedStats("stats") + .field("value") + .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params)) .sigma(sigma)) .execute().actionGet(); @@ -374,7 +377,11 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(extendedStats("stats").field("values").script(new Script("_value - 1")).sigma(sigma)) + .addAggregation( + extendedStats("stats") + .field("values") + .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null)) + .sigma(sigma)) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -401,9 +408,11 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - extendedStats("stats").field("values").script(new Script("_value - dec", ScriptType.INLINE, null, params)) + extendedStats("stats") + .field("values") + .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params)) .sigma(sigma)) - .execute().actionGet(); + .get(); assertHitCount(searchResponse, 10); @@ -426,7 +435,10 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(extendedStats("stats").script(new Script("doc['value'].value")).sigma(sigma)) + .addAggregation( + extendedStats("stats") + .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null)) + .sigma(sigma)) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -449,11 +461,16 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); + + Script script = new Script("doc['value'].value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params); + double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - extendedStats("stats").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params)).sigma(sigma)) + extendedStats("stats") + .script(script) + .sigma(sigma)) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -477,7 +494,10 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(extendedStats("stats").script(new Script("doc['values'].values")).sigma(sigma)) + .addAggregation( + extendedStats("stats") + .script(new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null)) + .sigma(sigma)) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -500,12 +520,16 @@ public class ExtendedStatsTests extends AbstractNumericTestCase { public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); + + Script script = new Script("[ doc['value'].value, doc['value'].value - dec ]", ScriptType.INLINE, + AggregationTestScriptsPlugin.NAME, params); + double sigma = randomDouble() * randomIntBetween(1, 10); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - extendedStats("stats").script( - new Script("[ doc['value'].value, doc['value'].value - dec ]", ScriptType.INLINE, null, params)) + extendedStats("stats") + .script(script) .sigma(sigma)) .execute().actionGet(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java similarity index 85% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java index 22af6dd486e..d4ac8835ee0 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java @@ -16,26 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.max.Max; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -50,10 +51,10 @@ import static org.hamcrest.Matchers.notNullValue; /** * */ -public class MaxTests extends AbstractNumericTestCase { +public class MaxIT extends AbstractNumericTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(AggregationTestScriptsPlugin.class); } @Override @@ -161,7 +162,10 @@ public class MaxTests extends AbstractNumericTestCase { public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").field("value").script(new Script("_value + 1"))) + .addAggregation( + max("max") + .field("value") + .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -178,8 +182,11 @@ public class MaxTests extends AbstractNumericTestCase { params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").field("value").script(new Script("_value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation( + max("max") + .field("value") + .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))) + .get(); assertHitCount(searchResponse, 10); @@ -208,8 +215,11 @@ public class MaxTests extends AbstractNumericTestCase { public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").field("values").script(new Script("_value + 1"))) - .execute().actionGet(); + .addAggregation( + max("max") + .field("values") + .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertHitCount(searchResponse, 10); @@ -225,8 +235,11 @@ public class MaxTests extends AbstractNumericTestCase { params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").field("values").script(new Script("_value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation( + max("max") + .field("values") + .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))) + .get(); assertHitCount(searchResponse, 10); @@ -240,7 +253,9 @@ public class MaxTests extends AbstractNumericTestCase { public void testScriptSingleValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").script(new Script("doc['value'].value"))) + .addAggregation( + max("max") + .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) .execute().actionGet(); assertHitCount(searchResponse, 10); @@ -255,10 +270,13 @@ public class MaxTests extends AbstractNumericTestCase { public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); + + Script script = new Script("doc['value'].value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation(max("max").script(script)) + .get(); assertHitCount(searchResponse, 10); @@ -272,8 +290,10 @@ public class MaxTests extends AbstractNumericTestCase { public void testScriptMultiValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(max("max").script(new Script("doc['values'].values"))) - .execute().actionGet(); + .addAggregation( + max("max") + .script(new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))) + .get(); assertHitCount(searchResponse, 10); @@ -287,10 +307,13 @@ public class MaxTests extends AbstractNumericTestCase { public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); + + Script script = new Script("[ doc['value'].value, doc['value'].value + inc ]", ScriptType.INLINE, + AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation( - max("max").script(new Script("[ doc['value'].value, doc['value'].value + inc ]", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation(max("max").script(script)) + .get(); assertHitCount(searchResponse, 10); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java similarity index 86% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java index f61aad9b137..56c12fbc77f 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java @@ -16,19 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.min.Min; import java.util.Collection; @@ -37,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -51,10 +51,10 @@ import static org.hamcrest.Matchers.notNullValue; /** * */ -public class MinTests extends AbstractNumericTestCase { +public class MinIT extends AbstractNumericTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(AggregationTestScriptsPlugin.class); } @Override @@ -163,8 +163,11 @@ public class MinTests extends AbstractNumericTestCase { public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(min("min").field("value").script(new Script("_value - 1"))) - .execute().actionGet(); + .addAggregation( + min("min") + .field("value") + .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertHitCount(searchResponse, 10); @@ -178,10 +181,13 @@ public class MinTests extends AbstractNumericTestCase { public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); + + Script script = new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(min("min").field("value").script(new Script("_value - dec", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation(min("min").field("value").script(script)) + .get(); assertHitCount(searchResponse, 10); @@ -210,7 +216,11 @@ public class MinTests extends AbstractNumericTestCase { public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(min("min").field("values").script(new Script("_value - 1"))).execute().actionGet(); + .addAggregation( + min("min") + .field("values") + .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertHitCount(searchResponse, 10); @@ -224,7 +234,11 @@ public class MinTests extends AbstractNumericTestCase { // test what happens when values arrive in reverse order since the min // aggregator is optimized to work on sorted values SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").field("values").script(new Script("_value * -1"))).execute().actionGet(); + .addAggregation( + min("min") + .field("values") + .script(new Script("_value * -1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertHitCount(searchResponse, 10); @@ -238,9 +252,12 @@ public class MinTests extends AbstractNumericTestCase { public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); + + Script script = new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").field("values").script(new Script("_value - dec", ScriptType.INLINE, null, params))).execute() - .actionGet(); + .addAggregation(min("min").field("values").script(script)) + .get(); assertHitCount(searchResponse, 10); @@ -252,8 +269,11 @@ public class MinTests extends AbstractNumericTestCase { @Override public void testScriptSingleValued() throws Exception { + Script script = new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").script(new Script("doc['value'].value"))).execute().actionGet(); + .addAggregation(min("min").script(script)) + .get(); assertHitCount(searchResponse, 10); @@ -267,9 +287,12 @@ public class MinTests extends AbstractNumericTestCase { public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); + + Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params))).execute() - .actionGet(); + .addAggregation(min("min").script(script)) + .get(); assertHitCount(searchResponse, 10); @@ -281,8 +304,10 @@ public class MinTests extends AbstractNumericTestCase { @Override public void testScriptMultiValued() throws Exception { + Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()); SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(min("min").script(new Script("doc['values'].values"))).execute().actionGet(); + .addAggregation(min("min").script(script)) + .get(); assertHitCount(searchResponse, 10); @@ -296,14 +321,12 @@ public class MinTests extends AbstractNumericTestCase { public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); + SearchResponse searchResponse = client() .prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation( - min("min") - .script(new Script( - "List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;", - ScriptType.INLINE, null, params))).execute().actionGet(); + .addAggregation(min("min").script(AggregationTestScriptsPlugin.DECREMENT_ALL_VALUES)) + .get(); assertHitCount(searchResponse, 10); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java similarity index 89% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java index a8eec2ede85..4e46a0b6a66 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java @@ -16,20 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase; import org.elasticsearch.search.aggregations.metrics.stats.Stats; import java.util.Collection; @@ -38,6 +37,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -51,13 +51,10 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -/** - * - */ -public class StatsTests extends AbstractNumericTestCase { +public class StatsIT extends AbstractNumericTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(AggregationTestScriptsPlugin.class); } @Override @@ -224,8 +221,11 @@ public class StatsTests extends AbstractNumericTestCase { public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").field("value").script(new Script("_value + 1"))) - .execute().actionGet(); + .addAggregation( + stats("stats") + .field("value") + .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertShardExecutionState(searchResponse, 0); @@ -247,8 +247,11 @@ public class StatsTests extends AbstractNumericTestCase { params.put("inc", 1); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").field("value").script(new Script("_value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation( + stats("stats") + .field("value") + .script(new Script("_value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))) + .get(); assertShardExecutionState(searchResponse, 0); @@ -289,8 +292,11 @@ public class StatsTests extends AbstractNumericTestCase { public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").field("values").script(new Script("_value - 1"))) - .execute().actionGet(); + .addAggregation( + stats("stats") + .field("values") + .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertShardExecutionState(searchResponse, 0); @@ -312,8 +318,11 @@ public class StatsTests extends AbstractNumericTestCase { params.put("dec", 1); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").field("values").script(new Script("_value - dec", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation( + stats("stats") + .field("values") + .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))) + .get(); assertShardExecutionState(searchResponse, 0); @@ -333,8 +342,10 @@ public class StatsTests extends AbstractNumericTestCase { public void testScriptSingleValued() throws Exception { SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").script(new Script("doc['value'].value"))) - .execute().actionGet(); + .addAggregation( + stats("stats") + .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); assertShardExecutionState(searchResponse, 0); @@ -354,10 +365,13 @@ public class StatsTests extends AbstractNumericTestCase { public void testScriptSingleValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("inc", 1); + + Script script = new Script("doc['value'].value + inc", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").script(new Script("doc['value'].value + inc", ScriptType.INLINE, null, params))) - .execute().actionGet(); + .addAggregation(stats("stats").script(script)) + .get(); assertShardExecutionState(searchResponse, 0); @@ -375,10 +389,12 @@ public class StatsTests extends AbstractNumericTestCase { @Override public void testScriptMultiValued() throws Exception { + Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()); + SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) - .addAggregation(stats("stats").script(new Script("doc['values'].values"))) - .execute().actionGet(); + .addAggregation(stats("stats").script(script)) + .get(); assertShardExecutionState(searchResponse, 0); @@ -398,12 +414,16 @@ public class StatsTests extends AbstractNumericTestCase { public void testScriptMultiValuedWithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); + + Script script = new Script("[ doc['value'].value, doc['value'].value - dec ]", ScriptType.INLINE, + AggregationTestScriptsPlugin.NAME, params); + SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - stats("stats").script( - new Script("[ doc['value'].value, doc['value'].value - dec ]", ScriptType.INLINE, null, params))) - .execute().actionGet(); + stats("stats") + .script(script)) + .get(); assertShardExecutionState(searchResponse, 0); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java similarity index 76% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java rename to core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java index 0f9279da8f2..3854d200116 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java @@ -17,32 +17,38 @@ * under the License. */ -package org.elasticsearch.messy.tests; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; +package org.elasticsearch.search.functionscore; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.ScoreAccessor; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -50,27 +56,54 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class FunctionScoreTests extends ESIntegTestCase { +public class FunctionScoreIT extends ESIntegTestCase { + static final String TYPE = "type"; static final String INDEX = "index"; @Override protected Collection> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + scripts.put("1", vars -> 1.0d); + scripts.put("get score value", vars -> ((ScoreAccessor) vars.get("_score")).doubleValue()); + scripts.put("return (doc['num'].value)", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Longs num = (ScriptDocValues.Longs) doc.get("num"); + return num.getValue(); + }); + scripts.put("doc['random_score']", vars -> { + Map doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles randomScore = (ScriptDocValues.Doubles) doc.get("random_score"); + return randomScore.getValue(); + }); + return scripts; + } } public void testScriptScoresNested() throws IOException { createIndex(INDEX); index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); refresh(); + + Script scriptOne = new Script("1", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script scriptTwo = new Script("get score value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client().search( searchRequest().source( searchSource().query( functionScoreQuery( functionScoreQuery( - functionScoreQuery(scriptFunction(new Script("1"))), - scriptFunction(new Script("_score.doubleValue()"))), - scriptFunction(new Script("_score.doubleValue()")) + functionScoreQuery(scriptFunction(scriptOne)), + scriptFunction(scriptTwo)), + scriptFunction(scriptTwo) ) ) ) @@ -83,10 +116,14 @@ public class FunctionScoreTests extends ESIntegTestCase { createIndex(INDEX); index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); refresh(); + + Script script = new Script("get score value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client().search( searchRequest().source( - searchSource().query(functionScoreQuery(scriptFunction(new Script("_score.doubleValue()")))).aggregation( - terms("score_agg").script(new Script("_score.doubleValue()"))) + searchSource() + .query(functionScoreQuery(scriptFunction(script))) + .aggregation(terms("score_agg").script(script)) ) ).actionGet(); assertSearchResponse(response); @@ -100,10 +137,17 @@ public class FunctionScoreTests extends ESIntegTestCase { refresh(); float score = randomFloat(); float minScore = randomFloat(); + + index(INDEX, TYPE, jsonBuilder().startObject() + .field("num", 2) + .field("random_score", score) // Pass the random score as a document field so that it can be extracted in the script + .endObject()); + refresh(); + ensureYellow(); + + Script script = new Script("doc['random_score']", ScriptType.INLINE, CustomScriptPlugin.NAME, null); SearchResponse searchResponse = client().search( - searchRequest().source( - searchSource().query( - functionScoreQuery(scriptFunction(new Script(Float.toString(score)))).setMinScore(minScore))) + searchRequest().source(searchSource().query(functionScoreQuery(scriptFunction(script)).setMinScore(minScore))) ).actionGet(); if (score < minScore) { assertThat(searchResponse.getHits().getTotalHits(), is(0L)); @@ -113,8 +157,8 @@ public class FunctionScoreTests extends ESIntegTestCase { searchResponse = client().search( searchRequest().source(searchSource().query(functionScoreQuery(new MatchAllQueryBuilder(), new FilterFunctionBuilder[] { - new FilterFunctionBuilder(scriptFunction(new Script(Float.toString(score)))), - new FilterFunctionBuilder(scriptFunction(new Script(Float.toString(score)))) + new FilterFunctionBuilder(scriptFunction(script)), + new FilterFunctionBuilder(scriptFunction(script)) }).scoreMode(FiltersFunctionScoreQuery.ScoreMode.AVG).setMinScore(minScore))) ).actionGet(); if (score < minScore) { @@ -133,7 +177,7 @@ public class FunctionScoreTests extends ESIntegTestCase { docs.add(client().prepareIndex(INDEX, TYPE, Integer.toString(i)).setSource("num", i + scoreOffset)); } indexRandom(true, docs); - Script script = new Script("return (doc['num'].value)"); + Script script = new Script("return (doc['num'].value)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); int numMatchingDocs = numDocs + scoreOffset - minScore; if (numMatchingDocs < 0) { numMatchingDocs = 0; diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index 09129072177..b10272329e5 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -19,15 +19,24 @@ package org.elasticsearch.search.suggest; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.ReduceSearchPhaseException; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder; import org.elasticsearch.search.suggest.phrase.Laplace; import org.elasticsearch.search.suggest.phrase.LinearInterpolation; @@ -42,6 +51,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -57,6 +67,7 @@ import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.anyOf; @@ -1108,6 +1119,234 @@ public class SuggestSearchIT extends ESIntegTestCase { // assertThat(total, lessThan(1000L)); // Takes many seconds without fix - just for debugging } + @Override + protected Collection> nodePlugins() { + return Collections.singleton(DummyTemplatePlugin.class); + } + + public static class DummyTemplatePlugin extends Plugin implements ScriptPlugin { + @Override + public ScriptEngineService getScriptEngineService(Settings settings) { + return new DummyTemplateScriptEngine(); + } + } + + public static class DummyTemplateScriptEngine implements ScriptEngineService { + + // The collate query setter is hard coded to use mustache, so lets lie in this test about the script plugin, + // which makes the collate code thinks mustache is evaluating the query. + public static final String NAME = "mustache"; + + @Override + public void close() throws IOException { + } + + @Override + public String getType() { + return NAME; + } + + @Override + public String getExtension() { + return NAME; + } + + @Override + public Object compile(String scriptName, String scriptSource, Map params) { + return scriptSource; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, Map params) { + String script = (String) compiledScript.compiled(); + for (Entry entry : params.entrySet()) { + script = script.replace("{{" + entry.getKey() + "}}", String.valueOf(entry.getValue())); + } + String result = script; + return new ExecutableScript() { + @Override + public void setNextVar(String name, Object value) { + throw new UnsupportedOperationException("setNextVar not supported"); + } + + @Override + public Object run() { + return new BytesArray(result); + } + }; + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, Map vars) { + throw new UnsupportedOperationException("search script not supported"); + } + + @Override + public boolean isInlineScriptEnabled() { + return true; + } + } + + public void testPhraseSuggesterCollate() throws InterruptedException, ExecutionException, IOException { + CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder() + .put(indexSettings()) + .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable. + .put("index.analysis.analyzer.text.tokenizer", "standard") + .putArray("index.analysis.analyzer.text.filter", "lowercase", "my_shingle") + .put("index.analysis.filter.my_shingle.type", "shingle") + .put("index.analysis.filter.my_shingle.output_unigrams", true) + .put("index.analysis.filter.my_shingle.min_shingle_size", 2) + .put("index.analysis.filter.my_shingle.max_shingle_size", 3)); + + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("title") + .field("type", "text") + .field("analyzer", "text") + .endObject() + .endObject() + .endObject() + .endObject(); + assertAcked(builder.addMapping("type1", mapping)); + ensureGreen(); + + List titles = new ArrayList<>(); + + titles.add("United States House of Representatives Elections in Washington 2006"); + titles.add("United States House of Representatives Elections in Washington 2005"); + titles.add("State"); + titles.add("Houses of Parliament"); + titles.add("Representative Government"); + titles.add("Election"); + + List builders = new ArrayList<>(); + for (String title: titles) { + builders.add(client().prepareIndex("test", "type1").setSource("title", title)); + } + indexRandom(true, builders); + + // suggest without collate + PhraseSuggestionBuilder suggest = phraseSuggestion("title") + .addCandidateGenerator(new DirectCandidateGeneratorBuilder("title") + .suggestMode("always") + .maxTermFreq(.99f) + .size(10) + .maxInspections(200) + ) + .confidence(0f) + .maxErrors(2f) + .shardSize(30000) + .size(10); + Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest); + assertSuggestionSize(searchSuggest, 0, 10, "title"); + + // suggest with collate + String filterString = XContentFactory.jsonBuilder() + .startObject() + .startObject("match_phrase") + .field("{{field}}", "{{suggestion}}") + .endObject() + .endObject() + .string(); + PhraseSuggestionBuilder filteredQuerySuggest = suggest.collateQuery(filterString); + filteredQuerySuggest.collateParams(Collections.singletonMap("field", "title")); + searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", filteredQuerySuggest); + assertSuggestionSize(searchSuggest, 0, 2, "title"); + + // collate suggest with no result (boundary case) + searchSuggest = searchSuggest("Elections of Representatives Parliament", "title", filteredQuerySuggest); + assertSuggestionSize(searchSuggest, 0, 0, "title"); + + NumShards numShards = getNumShards("test"); + + // collate suggest with bad query + String incorrectFilterString = XContentFactory.jsonBuilder() + .startObject() + .startObject("test") + .field("title", "{{suggestion}}") + .endObject() + .endObject() + .string(); + PhraseSuggestionBuilder incorrectFilteredSuggest = suggest.collateQuery(incorrectFilterString); + Map> namedSuggestion = new HashMap<>(); + namedSuggestion.put("my_title_suggestion", incorrectFilteredSuggest); + try { + searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion); + fail("Post query error has been swallowed"); + } catch(ElasticsearchException e) { + // expected + } + + // suggest with collation + String filterStringAsFilter = XContentFactory.jsonBuilder() + .startObject() + .startObject("match_phrase") + .field("title", "{{suggestion}}") + .endObject() + .endObject() + .string(); + + PhraseSuggestionBuilder filteredFilterSuggest = suggest.collateQuery(filterStringAsFilter); + searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", + filteredFilterSuggest); + assertSuggestionSize(searchSuggest, 0, 2, "title"); + + // collate suggest with bad query + String filterStr = XContentFactory.jsonBuilder() + .startObject() + .startObject("pprefix") + .field("title", "{{suggestion}}") + .endObject() + .endObject() + .string(); + + PhraseSuggestionBuilder in = suggest.collateQuery(filterStr); + try { + searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion); + fail("Post filter error has been swallowed"); + } catch(ElasticsearchException e) { + //expected + } + + // collate script failure due to no additional params + String collateWithParams = XContentFactory.jsonBuilder() + .startObject() + .startObject("{{query_type}}") + .field("{{query_field}}", "{{suggestion}}") + .endObject() + .endObject() + .string(); + + + PhraseSuggestionBuilder phraseSuggestWithNoParams = suggest.collateQuery(collateWithParams); + try { + searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion); + fail("Malformed query (lack of additional params) should fail"); + } catch (ElasticsearchException e) { + // expected + } + + // collate script with additional params + Map params = new HashMap<>(); + params.put("query_type", "match_phrase"); + params.put("query_field", "title"); + + PhraseSuggestionBuilder phraseSuggestWithParams = suggest.collateQuery(collateWithParams).collateParams(params); + searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", + phraseSuggestWithParams); + assertSuggestionSize(searchSuggest, 0, 2, "title"); + + // collate query request with prune set to true + PhraseSuggestionBuilder phraseSuggestWithParamsAndReturn = suggest.collateQuery(collateWithParams).collateParams(params) + .collatePrune(true); + searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", + phraseSuggestWithParamsAndReturn); + assertSuggestionSize(searchSuggest, 0, 10, "title"); + assertSuggestionPhraseCollateMatchExists(searchSuggest, "title", 2); + } + protected Suggest searchSuggest(String name, SuggestionBuilder suggestion) { return searchSuggest(null, name, suggestion); } diff --git a/distribution/build.gradle b/distribution/build.gradle index dcd5f170fc5..bfbf96b5d2a 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -72,7 +72,8 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each // See https://discuss.gradle.org/t/cross-project-task-dependencies-ordering-screws-up-finalizers/13190 project.configure(project.subprojects.findAll { it.name != 'integ-test-zip' }) { Project distribution -> distribution.afterEvaluate({ - distribution.integTest.mustRunAfter("${module.path}:integTest#stop") + // some integTest tasks will have multiple finalizers + distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }.getFinalizedBy() }) } // also want to make sure the module's integration tests run after the integ-test-zip (ie rest tests) diff --git a/distribution/deb/src/test/java/org/elasticsearch/test/rest/RestIT.java b/distribution/deb/src/test/java/org/elasticsearch/test/rest/RestIT.java index 2d90db2c5b7..94629641849 100644 --- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/RestIT.java +++ b/distribution/deb/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */ -public class RestIT extends ESRestTestCase { +public class RestIT extends ESClientYamlSuiteTestCase { public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/distribution/integ-test-zip/build.gradle b/distribution/integ-test-zip/build.gradle index 67f99aa884a..f1f3a2c26c5 100644 --- a/distribution/integ-test-zip/build.gradle +++ b/distribution/integ-test-zip/build.gradle @@ -36,4 +36,3 @@ publishing { } integTest.dependsOn buildZip - diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java new file mode 100644 index 00000000000..71ec1948662 --- /dev/null +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest; + +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Response; + +import java.io.IOException; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.startsWith; + +/** + * Tests for the "Location" header returned when returning {@code 201 CREATED}. + */ +public class CreatedLocationHeaderIT extends ESRestTestCase { + public void testCreate() throws IOException { + locationTestCase("PUT", "test/test/1"); + } + + public void testIndexWithId() throws IOException { + locationTestCase("PUT", "test/test/1"); + } + + public void testIndexWithoutId() throws IOException { + locationTestCase("POST", "test/test"); + } + + public void testUpsert() throws IOException { + locationTestCase(client().performRequest("POST", "test/test/1/_update", emptyMap(), new StringEntity("{" + + "\"doc\": {\"test\": \"test\"}," + + "\"doc_as_upsert\": true}"))); + } + + private void locationTestCase(String method, String url) throws IOException { + locationTestCase(client().performRequest(method, url, emptyMap(), new StringEntity("{\"test\": \"test\"}"))); + locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), new StringEntity("{\"test\": \"test\"}"))); + } + + private void locationTestCase(Response response) throws IOException { + assertEquals(201, response.getStatusLine().getStatusCode()); + String location = response.getHeader("Location"); + assertThat(location, startsWith("/test/test/")); + Response getResponse = client().performRequest("GET", location); + assertEquals(singletonMap("test", "test"), entityAsMap(getResponse).get("_source")); + } +} diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java index 2d90db2c5b7..94629641849 100644 --- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */ -public class RestIT extends ESRestTestCase { +public class RestIT extends ESClientYamlSuiteTestCase { public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RestIT.java b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RestIT.java index 2d90db2c5b7..94629641849 100644 --- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RestIT.java +++ b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */ -public class RestIT extends ESRestTestCase { +public class RestIT extends ESClientYamlSuiteTestCase { public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/distribution/tar/src/test/java/org/elasticsearch/test/rest/RestIT.java b/distribution/tar/src/test/java/org/elasticsearch/test/rest/RestIT.java index 2d90db2c5b7..94629641849 100644 --- a/distribution/tar/src/test/java/org/elasticsearch/test/rest/RestIT.java +++ b/distribution/tar/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */ -public class RestIT extends ESRestTestCase { +public class RestIT extends ESClientYamlSuiteTestCase { public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/distribution/zip/src/test/java/org/elasticsearch/test/rest/RestIT.java b/distribution/zip/src/test/java/org/elasticsearch/test/rest/RestIT.java index 2d90db2c5b7..94629641849 100644 --- a/distribution/zip/src/test/java/org/elasticsearch/test/rest/RestIT.java +++ b/distribution/zip/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */ -public class RestIT extends ESRestTestCase { +public class RestIT extends ESClientYamlSuiteTestCase { public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 224080c522c..7a9cd3313b2 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -168,12 +168,6 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu If set to `false`, will require all security groups to be present for the instance to be used for the discovery. Defaults to `true`. -`ping_timeout`:: - - How long to wait for existing EC2 nodes to reply during discovery. - Defaults to `3s`. If no unit like `ms`, `s` or `m` is specified, - milliseconds are used. - `node_cache_time`:: How long the list of hosts is cached to prevent further requests to the AWS API. @@ -243,9 +237,9 @@ filter instances with a tag key set to `stage`, and a value of `dev`. Several ta to be set for the instance to be included. One practical use for tag filtering is when an ec2 cluster contains many nodes that are not running elasticsearch. In -this case (particularly with high `ping_timeout` values) there is a risk that a new node's discovery phase will end -before it has found the cluster (which will result in it declaring itself master of a new cluster with the same name -- highly undesirable). Tagging elasticsearch ec2 nodes and then filtering by that tag will resolve this issue. +this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new node's discovery phase +will end before it has found the cluster (which will result in it declaring itself master of a new cluster with the same +name - highly undesirable). Tagging elasticsearch ec2 nodes and then filtering by that tag will resolve this issue. [[discovery-ec2-attributes]] ===== Automatic Node Attributes diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index 38dad90d1fc..f543bf92709 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -377,9 +377,9 @@ For example, setting `discovery.gce.tags` to `dev` will only filter instances ha set will require all of those tags to be set for the instance to be included. One practical use for tag filtering is when an GCE cluster contains many nodes that are not running -elasticsearch. In this case (particularly with high ping_timeout values) there is a risk that a new node's discovery -phase will end before it has found the cluster (which will result in it declaring itself master of a new cluster -with the same name - highly undesirable). Adding tag on elasticsearch GCE nodes and then filtering by that +elasticsearch. In this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new +node's discovery phase will end before it has found the cluster (which will result in it declaring itself master of a +new cluster with the same name - highly undesirable). Adding tag on elasticsearch GCE nodes and then filtering by that tag will resolve this issue. Add your tag when building the new instance: diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc index 7ed01d9dc7f..4ca249d31db 100644 --- a/docs/reference/aggregations/pipeline.asciidoc +++ b/docs/reference/aggregations/pipeline.asciidoc @@ -107,8 +107,7 @@ a metric embedded inside a sibling aggregation: === Special Paths Instead of pathing to a metric, `buckets_path` can use a special `"_count"` path. This instructs -the pipeline aggregation to use the document count as it's input. For example, a moving average can be calculated on the document -count of each bucket, instead of a specific metric: +the pipeline aggregation to use the document count as it's input. For example, a moving average can be calculated on the document count of each bucket, instead of a specific metric: [source,js] -------------------------------------------------- @@ -128,6 +127,44 @@ count of each bucket, instead of a specific metric: -------------------------------------------------- <1> By using `_count` instead of a metric name, we can calculate the moving average of document counts in the histogram +The `buckets_path` can also use `"_bucket_count"` and path to a multi-bucket aggregation to use the number of buckets +returned by that aggregation in the pipeline aggregation instead of a metric. for example a `bucket_selector` can be +used here to filter out buckets which contain no buckets for an inner terms aggregation: + +[source,js] +-------------------------------------------------- +{ + "size": 0, + "aggs": { + "histo": { + "date_histogram": { + "field": "date", + "interval": "day" + }, + "aggs": { + "categories": { + "terms": { + "field": "category" + } + }, + "min_bucket_selector": { + "bucket_selector": { + "buckets_path": { + "count": "categories._bucket_count" <1> + }, + "script": { + "inline": "count != 0" + } + } + } + } + } + } +} +-------------------------------------------------- +<1> By using `_bucket_count` instead of a metric name, we can filter out `histo` buckets where they contain no buckets +for the `categories` aggregation + [[dots-in-agg-names]] [float] === Dealing with dots in agg names diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc index da0a5776ad0..76ee65a6abe 100644 --- a/docs/reference/migration/migrate_5_0/settings.asciidoc +++ b/docs/reference/migration/migrate_5_0/settings.asciidoc @@ -28,6 +28,9 @@ The `name` setting has been removed and is replaced by `node.name`. Usage of The `node.add_id_to_custom_path` was renamed to `add_lock_id_to_custom_path`. +The default for the `node.name` settings is now the first 7 charachters of the node id, +which is in turn a randomly generated UUID. + The settings `node.mode` and `node.local` are removed. Local mode should be configured via `discovery.type: local` and `transport.type:local`. In order to disable _http_ please use `http.enabled: false` diff --git a/docs/reference/modules/indices/circuit_breaker.asciidoc b/docs/reference/modules/indices/circuit_breaker.asciidoc index 762833d527b..fed1c350274 100644 --- a/docs/reference/modules/indices/circuit_breaker.asciidoc +++ b/docs/reference/modules/indices/circuit_breaker.asciidoc @@ -47,7 +47,7 @@ request) from exceeding a certain amount of memory. `indices.breaker.request.limit`:: - Limit for request breaker, defaults to 40% of JVM heap + Limit for request breaker, defaults to 60% of JVM heap `indices.breaker.request.overhead`:: @@ -74,4 +74,3 @@ memory on a node. The memory usage is based on the content length of the request [[http-circuit-breaker]] [float] - diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index e3be0b780c4..aecd205b613 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -70,9 +70,9 @@ environments, otherwise you might end up with nodes joining the wrong cluster. [[node.name]] === `node.name` -By default, Elasticsearch will randomly pick a descriptive `node.name` from a -list of around 3000 Marvel characters when your node starts up, but this also -means that the `node.name` will change the next time the node restarts. +By default, Elasticsearch will take the 7 first charachter of the randomly generated uuid used as the node id. +Note that the node id is persisted and does not change when a node restarts and therefore the default node name +will also not change. It is worth configuring a more meaningful name which will also have the advantage of persisting after restarting the node: diff --git a/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java b/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java index 452bea4a647..d8eccda61a2 100644 --- a/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java +++ b/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java @@ -22,14 +22,14 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; import java.util.List; -public class SmokeTestDocsIT extends ESRestTestCase { +public class SmokeTestDocsIT extends ESClientYamlSuiteTestCase { public SmokeTestDocsIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -37,7 +37,7 @@ public class SmokeTestDocsIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } @Override diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java index 69ddd46edc2..47609494a61 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java @@ -20,7 +20,7 @@ package org.elasticsearch.search.aggregations.matrix; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; @@ -29,13 +29,13 @@ import java.io.IOException; /** * */ -public class MatrixAggregationRestIT extends ESRestTestCase { +public class MatrixAggregationRestIT extends ESClientYamlSuiteTestCase { public MatrixAggregationRestIT(@Name("yaml")RestTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java index 94418ac552c..65ee917f30e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.ingest.common; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class IngestCommonRestIT extends ESRestTestCase { +public class IngestCommonRestIT extends ESClientYamlSuiteTestCase { public IngestCommonRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class IngestCommonRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml index 464199f6d3b..a4395427e04 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml @@ -19,6 +19,11 @@ "target_field" : "date", "formats" : ["yyyy"] } + }, + { + "uppercase" : { + "field": "field1" + } } ], "on_failure" : [ diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java index 3da22bf8a43..8ff5168a7cf 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.script.expression; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class ExpressionRestIT extends ESRestTestCase { +public class ExpressionRestIT extends ESClientYamlSuiteTestCase { public ExpressionRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class ExpressionRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java index 867e6d29c88..383e7dd6028 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java @@ -42,18 +42,12 @@ ^^^^^ note: the methods from this test using mustache were moved to the mustache module under its messy tests package. renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java renamed: core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java renamed: core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java renamed: core/src/test/java/org/elasticsearch/script/IndexLookupIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java renamed: core/src/test/java/org/elasticsearch/script/IndexedScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java renamed: core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/InnerHitsTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java renamed: core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/PercolatorTests.java renamed: core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java @@ -65,8 +59,6 @@ renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SignificantTermsSignificanceScoreTests.java renamed: core/src/test/java/org/elasticsearch/nested/SimpleNestedIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleNestedTests.java renamed: core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SumTests.java renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/TopHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TopHitsTests.java renamed: core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java index b73ec250daf..b681e5f6857 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.script.groovy; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class GroovyRestIT extends ESRestTestCase { +public class GroovyRestIT extends ESClientYamlSuiteTestCase { public GroovyRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class GroovyRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index 271faa23608..2d34eacc572 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -20,7 +20,7 @@ esplugin { description 'Mustache scripting integration for Elasticsearch' classname 'org.elasticsearch.script.mustache.MustachePlugin' - hasClientJar = true // For the template query + hasClientJar = true // For the template apis and query } dependencies { @@ -31,9 +31,6 @@ integTest { cluster { setting 'script.inline', 'true' setting 'script.stored', 'true' - - - File template = new File('src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache') - extraConfigFile 'scripts/template_1.mustache', template + setting 'path.scripts', "${project.buildDir}/resources/test/templates" } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java similarity index 96% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java index f5ab172f8ff..99eb9709f73 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java similarity index 98% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index 6fce2b2d492..b1db44defa0 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java similarity index 98% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java index 5b0f802ab83..4624e8caa24 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java similarity index 99% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java index f9e99ffc7e3..6201e9305ef 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 5d7b3b4fb56..170070564f9 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -21,22 +21,12 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.search.template.MultiSearchTemplateAction; -import org.elasticsearch.action.search.template.SearchTemplateAction; -import org.elasticsearch.action.search.template.TransportMultiSearchTemplateAction; -import org.elasticsearch.action.search.template.TransportSearchTemplateAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.rest.action.search.template.RestDeleteSearchTemplateAction; -import org.elasticsearch.rest.action.search.template.RestGetSearchTemplateAction; -import org.elasticsearch.rest.action.search.template.RestMultiSearchTemplateAction; -import org.elasticsearch.rest.action.search.template.RestPutSearchTemplateAction; -import org.elasticsearch.rest.action.search.template.RestRenderSearchTemplateAction; -import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction; import org.elasticsearch.script.ScriptEngineService; import java.util.Arrays; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java similarity index 96% rename from modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java index 075e6431b4b..ed4506686d8 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.rest.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java similarity index 96% rename from modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java index b2c44e10a48..47ab373b261 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.rest.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java similarity index 93% rename from modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index add4719ad93..c0ed4b59ef6 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -17,13 +17,10 @@ * under the License. */ -package org.elasticsearch.rest.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.action.search.template.MultiSearchTemplateAction; -import org.elasticsearch.action.search.template.MultiSearchTemplateRequest; -import org.elasticsearch.action.search.template.SearchTemplateRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java similarity index 96% rename from modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java index cca88fd4ecd..71530eb4409 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.rest.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java similarity index 92% rename from modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java index 5e7e75e9c81..33c8a17c742 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java @@ -17,10 +17,8 @@ * under the License. */ -package org.elasticsearch.rest.action.search.template; +package org.elasticsearch.script.mustache; -import org.elasticsearch.action.search.template.SearchTemplateAction; -import org.elasticsearch.action.search.template.SearchTemplateRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java similarity index 96% rename from modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index e12137ee3c5..42cd3987d47 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -17,12 +17,10 @@ * under the License. */ -package org.elasticsearch.rest.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.template.SearchTemplateAction; -import org.elasticsearch.action.search.template.SearchTemplateRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java similarity index 93% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java index a3501f51bb1..2982fbd70c6 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java @@ -17,10 +17,9 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.Action; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.ElasticsearchClient; public class SearchTemplateAction extends Action { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java similarity index 99% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java index ff3d97401a3..a4a65336046 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java similarity index 97% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java index 3d44535d80b..811c2523e06 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.search.SearchRequest; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java similarity index 98% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java index 86b12a0fe56..f4647b713de 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.search.SearchResponse; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java similarity index 98% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java index 642fe7648da..a613f15972b 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java similarity index 97% rename from modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index 0c55a63596a..ded396deb12 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; @@ -36,7 +36,6 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggesters; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java deleted file mode 100644 index 1e352e1ab4c..00000000000 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.messy.tests; - -import org.elasticsearch.action.search.template.SearchTemplateRequestBuilder; -import org.elasticsearch.action.search.template.SearchTemplateResponse; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.env.Environment; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.mustache.MustachePlugin; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; - -@ESIntegTestCase.SuiteScopeTestCase -public class RenderSearchTemplateTests extends ESIntegTestCase { - private static final String TEMPLATE_CONTENTS = "{\"size\":\"{{size}}\",\"query\":{\"match\":{\"foo\":\"{{value}}\"}},\"aggs\":{\"objects\":{\"terms\":{\"field\":\"{{value}}\",\"size\":\"{{size}}\"}}}}"; - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(MustachePlugin.class); - } - - @Override - protected Collection> transportClientPlugins() { - return nodePlugins(); - } - - @Override - protected void setupSuiteScopeCluster() throws Exception { - ElasticsearchAssertions.assertAcked(client().admin().cluster().preparePutStoredScript() - .setScriptLang(MustacheScriptEngineService.NAME) - .setId("index_template_1") - .setSource(new BytesArray("{ \"template\": " + TEMPLATE_CONTENTS + " }"))); - } - - @Override - public Settings nodeSettings(int nodeOrdinal) { - Path configDir = createTempDir(); - Path scriptsDir = configDir.resolve("scripts"); - try { - Files.createDirectories(scriptsDir); - Files.write(scriptsDir.resolve("file_template_1.mustache"), TEMPLATE_CONTENTS.getBytes("UTF-8")); - } catch (Exception e) { - throw new RuntimeException(e); - } - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_CONF_SETTING.getKey(), configDir).build(); - } - - public void testInlineTemplate() throws ExecutionException, InterruptedException { - Map params = new HashMap<>(); - params.put("value", "bar"); - params.put("size", 20); - SearchTemplateResponse response = prepareRenderSearchTemplate(TEMPLATE_CONTENTS, ScriptType.INLINE, params).get(); - assertThat(response, notNullValue()); - assertFalse(response.hasResponse()); - BytesReference source = response.getSource(); - assertThat(source, notNullValue()); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); - assertThat(sourceAsMap, notNullValue()); - String expected = TEMPLATE_CONTENTS.replace("{{value}}", "bar").replace("{{size}}", "20"); - Map expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); - assertThat(sourceAsMap, equalTo(expectedMap)); - - params = new HashMap<>(); - params.put("value", "baz"); - params.put("size", 100); - response = prepareRenderSearchTemplate(TEMPLATE_CONTENTS, ScriptType.INLINE, params).get(); - assertThat(response, notNullValue()); - assertFalse(response.hasResponse()); - source = response.getSource(); - assertThat(source, notNullValue()); - sourceAsMap = XContentHelper.convertToMap(source, false).v2(); - expected = TEMPLATE_CONTENTS.replace("{{value}}", "baz").replace("{{size}}", "100"); - expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); - assertThat(sourceAsMap, equalTo(expectedMap)); - } - - public void testIndexedTemplate() throws ExecutionException, InterruptedException { - Map params = new HashMap<>(); - params.put("value", "bar"); - params.put("size", 20); - SearchTemplateResponse response = prepareRenderSearchTemplate("index_template_1", ScriptType.STORED, params).get(); - assertThat(response, notNullValue()); - assertFalse(response.hasResponse()); - BytesReference source = response.getSource(); - assertThat(source, notNullValue()); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); - assertThat(sourceAsMap, notNullValue()); - String expected = TEMPLATE_CONTENTS.replace("{{value}}", "bar").replace("{{size}}", "20"); - Map expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); - assertThat(sourceAsMap, equalTo(expectedMap)); - - params = new HashMap<>(); - params.put("value", "baz"); - params.put("size", 100); - response = prepareRenderSearchTemplate("index_template_1", ScriptType.STORED, params).get(); - assertThat(response, notNullValue()); - source = response.getSource(); - assertThat(source, notNullValue()); - sourceAsMap = XContentHelper.convertToMap(source, false).v2(); - expected = TEMPLATE_CONTENTS.replace("{{value}}", "baz").replace("{{size}}", "100"); - expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); - assertThat(sourceAsMap, equalTo(expectedMap)); - } - - public void testFileTemplate() throws ExecutionException, InterruptedException { - Map params = new HashMap<>(); - params.put("value", "bar"); - params.put("size", 20); - SearchTemplateResponse response = prepareRenderSearchTemplate("file_template_1", ScriptType.FILE, params).get(); - assertThat(response, notNullValue()); - assertFalse(response.hasResponse()); - BytesReference source = response.getSource(); - assertThat(source, notNullValue()); - Map sourceAsMap = XContentHelper.convertToMap(source, false).v2(); - assertThat(sourceAsMap, notNullValue()); - String expected = TEMPLATE_CONTENTS.replace("{{value}}", "bar").replace("{{size}}", "20"); - Map expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); - assertThat(sourceAsMap, equalTo(expectedMap)); - - params = new HashMap<>(); - params.put("value", "baz"); - params.put("size", 100); - response = prepareRenderSearchTemplate("file_template_1", ScriptType.FILE, params).get(); - assertThat(response, notNullValue()); - source = response.getSource(); - assertThat(source, notNullValue()); - sourceAsMap = XContentHelper.convertToMap(source, false).v2(); - expected = TEMPLATE_CONTENTS.replace("{{value}}", "baz").replace("{{size}}", "100"); - expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2(); - assertThat(sourceAsMap, equalTo(expectedMap)); - } - - private SearchTemplateRequestBuilder prepareRenderSearchTemplate(String script, ScriptType type, Map params) { - return new SearchTemplateRequestBuilder(client()).setScript(script).setScriptType(type).setScriptParams(params).setSimulate(true); - } -} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java deleted file mode 100644 index bac598ba282..00000000000 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.messy.tests; - - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.mustache.MustachePlugin; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder; -import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; -import org.elasticsearch.test.ESIntegTestCase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ExecutionException; - -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize; -import static org.hamcrest.Matchers.equalTo; - -/** - * Integration tests for term and phrase suggestions. Many of these tests many requests that vary only slightly from one another. Where - * possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that - * request, modify again, request again, etc. This makes it very obvious what changes between requests. - */ -public class SuggestSearchTests extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(MustachePlugin.class); - } - - public void testPhraseSuggesterCollate() throws InterruptedException, ExecutionException, IOException { - CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder() - .put(indexSettings()) - .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable. - .put("index.analysis.analyzer.text.tokenizer", "standard") - .putArray("index.analysis.analyzer.text.filter", "lowercase", "my_shingle") - .put("index.analysis.filter.my_shingle.type", "shingle") - .put("index.analysis.filter.my_shingle.output_unigrams", true) - .put("index.analysis.filter.my_shingle.min_shingle_size", 2) - .put("index.analysis.filter.my_shingle.max_shingle_size", 3)); - - XContentBuilder mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("title") - .field("type", "text") - .field("analyzer", "text") - .endObject() - .endObject() - .endObject() - .endObject(); - assertAcked(builder.addMapping("type1", mapping)); - ensureGreen(); - - List titles = new ArrayList<>(); - - titles.add("United States House of Representatives Elections in Washington 2006"); - titles.add("United States House of Representatives Elections in Washington 2005"); - titles.add("State"); - titles.add("Houses of Parliament"); - titles.add("Representative Government"); - titles.add("Election"); - - List builders = new ArrayList<>(); - for (String title: titles) { - builders.add(client().prepareIndex("test", "type1").setSource("title", title)); - } - indexRandom(true, builders); - - // suggest without collate - PhraseSuggestionBuilder suggest = phraseSuggestion("title") - .addCandidateGenerator(new DirectCandidateGeneratorBuilder("title") - .suggestMode("always") - .maxTermFreq(.99f) - .size(10) - .maxInspections(200) - ) - .confidence(0f) - .maxErrors(2f) - .shardSize(30000) - .size(10); - Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest); - assertSuggestionSize(searchSuggest, 0, 10, "title"); - - // suggest with collate - String filterString = XContentFactory.jsonBuilder() - .startObject() - .startObject("match_phrase") - .field("{{field}}", "{{suggestion}}") - .endObject() - .endObject() - .string(); - PhraseSuggestionBuilder filteredQuerySuggest = suggest.collateQuery(filterString); - filteredQuerySuggest.collateParams(Collections.singletonMap("field", "title")); - searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", filteredQuerySuggest); - assertSuggestionSize(searchSuggest, 0, 2, "title"); - - // collate suggest with no result (boundary case) - searchSuggest = searchSuggest("Elections of Representatives Parliament", "title", filteredQuerySuggest); - assertSuggestionSize(searchSuggest, 0, 0, "title"); - - NumShards numShards = getNumShards("test"); - - // collate suggest with bad query - String incorrectFilterString = XContentFactory.jsonBuilder() - .startObject() - .startObject("test") - .field("title", "{{suggestion}}") - .endObject() - .endObject() - .string(); - PhraseSuggestionBuilder incorrectFilteredSuggest = suggest.collateQuery(incorrectFilterString); - Map> namedSuggestion = new HashMap<>(); - namedSuggestion.put("my_title_suggestion", incorrectFilteredSuggest); - try { - searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion); - fail("Post query error has been swallowed"); - } catch(ElasticsearchException e) { - // expected - } - - // suggest with collation - String filterStringAsFilter = XContentFactory.jsonBuilder() - .startObject() - .startObject("match_phrase") - .field("title", "{{suggestion}}") - .endObject() - .endObject() - .string(); - - PhraseSuggestionBuilder filteredFilterSuggest = suggest.collateQuery(filterStringAsFilter); - searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", - filteredFilterSuggest); - assertSuggestionSize(searchSuggest, 0, 2, "title"); - - // collate suggest with bad query - String filterStr = XContentFactory.jsonBuilder() - .startObject() - .startObject("pprefix") - .field("title", "{{suggestion}}") - .endObject() - .endObject() - .string(); - - PhraseSuggestionBuilder in = suggest.collateQuery(filterStr); - try { - searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion); - fail("Post filter error has been swallowed"); - } catch(ElasticsearchException e) { - //expected - } - - // collate script failure due to no additional params - String collateWithParams = XContentFactory.jsonBuilder() - .startObject() - .startObject("{{query_type}}") - .field("{{query_field}}", "{{suggestion}}") - .endObject() - .endObject() - .string(); - - - PhraseSuggestionBuilder phraseSuggestWithNoParams = suggest.collateQuery(collateWithParams); - try { - searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion); - fail("Malformed query (lack of additional params) should fail"); - } catch (ElasticsearchException e) { - // expected - } - - // collate script with additional params - Map params = new HashMap<>(); - params.put("query_type", "match_phrase"); - params.put("query_field", "title"); - - PhraseSuggestionBuilder phraseSuggestWithParams = suggest.collateQuery(collateWithParams).collateParams(params); - searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", - phraseSuggestWithParams); - assertSuggestionSize(searchSuggest, 0, 2, "title"); - - // collate query request with prune set to true - PhraseSuggestionBuilder phraseSuggestWithParamsAndReturn = suggest.collateQuery(collateWithParams).collateParams(params) - .collatePrune(true); - searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", - phraseSuggestWithParamsAndReturn); - assertSuggestionSize(searchSuggest, 0, 10, "title"); - assertSuggestionPhraseCollateMatchExists(searchSuggest, "title", 2); - } - - protected Suggest searchSuggest(String name, SuggestionBuilder suggestion) { - return searchSuggest(null, name, suggestion); - } - - protected Suggest searchSuggest(String suggestText, String name, SuggestionBuilder suggestion) { - Map> map = new HashMap<>(); - map.put(name, suggestion); - return searchSuggest(suggestText, 0, map); - } - - protected Suggest searchSuggest(String suggestText, int expectShardsFailed, Map> suggestions) { - SearchRequestBuilder builder = client().prepareSearch().setSize(0); - SuggestBuilder suggestBuilder = new SuggestBuilder(); - if (suggestText != null) { - suggestBuilder.setGlobalText(suggestText); - } - for (Entry> suggestion : suggestions.entrySet()) { - suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue()); - } - builder.suggest(suggestBuilder); - SearchResponse actionGet = builder.execute().actionGet(); - assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); - return actionGet.getSuggest(); - } -} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java deleted file mode 100644 index 9c09e9245f6..00000000000 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.messy.tests; - -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.util.Accountable; -import org.elasticsearch.Version; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; -import org.elasticsearch.common.inject.multibindings.Multibinder; -import org.elasticsearch.common.inject.util.Providers; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.mustache.TemplateQueryBuilder; -import org.elasticsearch.index.query.functionscore.ScoreFunctionParser; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; -import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; -import org.elasticsearch.script.ScriptModule; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.IndexSettingsModule; -import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.threadpool.ThreadPool; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.lang.reflect.Proxy; -import java.util.Collections; -import java.util.List; -import java.util.function.Supplier; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.containsString; - -/** - * Test parsing and executing a template request. - */ -// NOTE: this can't be migrated to ESSingleNodeTestCase because of the custom path.conf -public class TemplateQueryParserTests extends ESTestCase { - - private Injector injector; - private Supplier contextFactory; - - @Before - public void setup() throws IOException { - Settings settings = Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put(Environment.PATH_CONF_SETTING.getKey(), this.getDataPath("config")) - .put("node.name", getClass().getName()) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) - .build(); - Environment environment = new Environment(settings); - final Client proxy = (Client) Proxy.newProxyInstance( - Client.class.getClassLoader(), - new Class[]{Client.class}, (proxy1, method, args) -> { - throw new UnsupportedOperationException("client is just a dummy"); - }); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - Index index = idxSettings.getIndex(); - // TODO: make this use a mock engine instead of mustache and it will no longer be messy! - ScriptModule scriptModule = new ScriptModule(settings, environment, null, singletonList(new MustacheScriptEngineService(settings)), - emptyList()); - List> scriptSettings = scriptModule.getSettings(); - scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED); - SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList()); - injector = new ModulesBuilder().add( - (b) -> { - b.bind(ThreadPool.class).toInstance(new ThreadPool(settings)); - b.bind(Client.class).toInstance(proxy); // not needed here - Multibinder.newSetBinder(b, ScoreFunctionParser.class); - b.bind(ClusterService.class).toProvider(Providers.of((ClusterService) null)); - b.bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class); - }, - settingsModule, - new SearchModule(settings, new NamedWriteableRegistry(), false, emptyList()) { - @Override - protected void configureSearch() { - // skip so we don't need transport - } - }, - new IndexSettingsModule(index, settings) - ).createInjector(); - - AnalysisService analysisService = createAnalysisService(idxSettings, settings); - SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); - MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()).getMapperRegistry(); - MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () -> - contextFactory.get()); - IndicesFieldDataCache cache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); - IndexFieldDataService indexFieldDataService = new IndexFieldDataService(idxSettings, cache, injector.getInstance(CircuitBreakerService.class), mapperService); - BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() { - @Override - public void onCache(ShardId shardId, Accountable accountable) {} - @Override - public void onRemoval(ShardId shardId, Accountable accountable) {} - }); - IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); - contextFactory = () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, - similarityService, scriptModule.getScriptService(), indicesQueriesRegistry, proxy, null, null); - } - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - terminate(injector.getInstance(ThreadPool.class)); - } - - public void testParser() throws IOException { - String templateString = "{" + "\"inline\":{\"match_{{template}}\": {}}," + "\"params\":{\"template\":\"all\"}" + "}"; - - XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString); - QueryShardContext context = contextFactory.get(); - templateSourceParser.nextToken(); - - Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get(), - context).toQuery(context); - assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery); - } - - public void testParseTemplateAsSingleStringWithConditionalClause() throws IOException { - String templateString = "{" + " \"inline\" : \"{ \\\"match_{{#use_it}}{{template}}{{/use_it}}\\\":{} }\"," + " \"params\":{" - + " \"template\":\"all\"," + " \"use_it\": true" + " }" + "}"; - XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString); - QueryShardContext context = contextFactory.get(); - - Query query = QueryBuilder - .rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get(), context) - .toQuery(context); - assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery); - } - - /** - * Test that the template query parser can parse and evaluate template - * expressed as a single string but still it expects only the query - * specification (thus this test should fail with specific exception). - */ - public void testParseTemplateFailsToParseCompleteQueryAsSingleString() throws IOException { - String templateString = "{" + " \"inline\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\"," - + " \"params\":{" + " \"size\":2" + " }\n" + "}"; - - XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString); - QueryShardContext context = contextFactory.get(); - - try { - TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get().rewrite(context); - fail("Expected ParsingException"); - } catch (ParsingException e) { - assertThat(e.getMessage(), containsString("query malformed, no field after start_object")); - } - } - - public void testParserCanExtractTemplateNames() throws Exception { - String templateString = "{ \"file\": \"storedTemplate\" ,\"params\":{\"template\":\"all\" } } "; - - XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString); - QueryShardContext context = contextFactory.get(); - templateSourceParser.nextToken(); - - - Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get(), - context).toQuery(context); - assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery); - } - - public void testMustRewrite() throws Exception { - String templateString = "{ \"file\": \"storedTemplate\" ,\"params\":{\"template\":\"all\" } } "; - - XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString); - QueryShardContext context = contextFactory.get(); - templateSourceParser.nextToken(); - try { - TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get().toQuery(context); - fail(); - } catch (UnsupportedOperationException ex) { - assertEquals("this query must be rewritten first", ex.getMessage()); - } - } -} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java deleted file mode 100644 index a2325b2d511..00000000000 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * This package contains tests that use mustache to test what looks - * to be unrelated functionality, or functionality that should be - * tested with a mock instead. Instead of doing an epic battle - * with these tests, they are temporarily moved here to the mustache - * module's tests, but that is likely not where they belong. Please - * help by cleaning them up and we can remove this package! - * - *
    - *
  • If the test is actually testing mustache specifically, move to - * the org.elasticsearch.script.mustache tests package of this module
    • - *
  • If the test is testing templating integration with another core subsystem, - * fix it to use a mock instead, so it can be in the core tests again
    • - *
  • If the test is just being lazy, and does not really need templating to test - * something, clean it up!
    • - *
- */ -/* List of renames that took place: -renamed: core/src/test/java/org/elasticsearch/validate/RenderSearchTemplateIT.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java -renamed: core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java -renamed: core/src/test/java/org/elasticsearch/index/query/TemplateQueryParserTests.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java -renamed: core/src/test/java/org/elasticsearch/index/query/TemplateQueryIT.java -> modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java -renamed: core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java -> module/lang-mustache/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java - ^^^^^ note: just the methods from this test using mustache were moved here, the others use groovy and are in the groovy module under its messy tests package. -renamed: rest-api-spec/test/msearch/10_basic.yaml -> module/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml - */ - -package org.elasticsearch.messy.tests; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java similarity index 98% rename from modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java index 03d04e518dc..f5d1a9dd791 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java @@ -17,14 +17,13 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.test.ESIntegTestCase; import java.util.Arrays; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java similarity index 95% rename from modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java index 4724a758107..d7807c1a268 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java @@ -17,11 +17,10 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.search.template.RestMultiSearchTemplateAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.StreamsUtils; @@ -34,7 +33,7 @@ import static org.hamcrest.Matchers.nullValue; public class MultiSearchTemplateRequestTests extends ESTestCase { public void testParseRequest() throws Exception { - byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/template/simple-msearch-template.json"); + byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/script/mustache/simple-msearch-template.json"); RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build(); MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java index 727d0c4316d..95f86dec3c7 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.script.mustache; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class MustacheRestIT extends ESRestTestCase { +public class MustacheRestIT extends ESClientYamlSuiteTestCase { public MustacheRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class MustacheRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index bdba0346a85..693ada174b9 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -19,9 +19,14 @@ package org.elasticsearch.script.mustache; import com.github.mustachejava.MustacheFactory; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -54,7 +59,8 @@ public class MustacheScriptEngineTests extends ESTestCase { + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}" + "}}, \"negative_boost\": {{boost_val}} } }}"; Map vars = new HashMap<>(); vars.put("boost_val", "0.3"); - BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run(); + BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", + qe.compile(null, template, compileParams)), vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}", o.utf8ToString()); @@ -65,13 +71,35 @@ public class MustacheScriptEngineTests extends ESTestCase { Map vars = new HashMap<>(); vars.put("boost_val", "0.3"); vars.put("body_val", "\"quick brown\""); - BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run(); + BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", + qe.compile(null, template, compileParams)), vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}", o.utf8ToString()); } } + public void testSimple() throws IOException { + String templateString = "{" + "\"inline\":{\"match_{{template}}\": {}}," + "\"params\":{\"template\":\"all\"}" + "}"; + XContentParser parser = XContentFactory.xContent(templateString).createParser(templateString); + Script script = Script.parse(parser, new ParseFieldMatcher(false)); + CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, null, "mustache", + qe.compile(null, script.getScript(), Collections.emptyMap())); + ExecutableScript executableScript = qe.executable(compiledScript, script.getParams()); + assertThat(((BytesReference) executableScript.run()).utf8ToString(), equalTo("{\"match_all\":{}}")); + } + + public void testParseTemplateAsSingleStringWithConditionalClause() throws IOException { + String templateString = "{" + " \"inline\" : \"{ \\\"match_{{#use_it}}{{template}}{{/use_it}}\\\":{} }\"," + " \"params\":{" + + " \"template\":\"all\"," + " \"use_it\": true" + " }" + "}"; + XContentParser parser = XContentFactory.xContent(templateString).createParser(templateString); + Script script = Script.parse(parser, new ParseFieldMatcher(false)); + CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, null, "mustache", + qe.compile(null, script.getScript(), Collections.emptyMap())); + ExecutableScript executableScript = qe.executable(compiledScript, script.getParams()); + assertThat(((BytesReference) executableScript.run()).utf8ToString(), equalTo("{ \"match_all\":{} }")); + } + public void testEscapeJson() throws IOException { { StringWriter writer = new StringWriter(); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index 91098eb1c88..9b48afe834a 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -44,7 +44,6 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.script.ScriptService.ScriptType.INLINE; import static org.elasticsearch.script.mustache.MustacheScriptEngineService.CONTENT_TYPE_PARAM; -import static org.elasticsearch.script.mustache.MustacheScriptEngineService.JSON_CONTENT_TYPE; import static org.elasticsearch.script.mustache.MustacheScriptEngineService.PLAIN_TEXT_CONTENT_TYPE; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; @@ -147,7 +146,6 @@ public class MustacheTests extends ESTestCase { public void testEscaping() { // json string escaping enabled: - Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE); Mustache mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.emptyMap()); CompiledScript compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache); ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\"")); @@ -156,7 +154,8 @@ public class MustacheTests extends ESTestCase { assertThat(result, equalTo("{ \"field1\": \"a \\\"value\\\"\"}")); // json string escaping disabled: - mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.singletonMap(CONTENT_TYPE_PARAM, PLAIN_TEXT_CONTENT_TYPE)); + mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", + Collections.singletonMap(CONTENT_TYPE_PARAM, PLAIN_TEXT_CONTENT_TYPE)); compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache); executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\"")); rawResult = (BytesReference) executableScript.run(); @@ -236,10 +235,12 @@ public class MustacheTests extends ESTestCase { Map ctx = Collections.singletonMap("ctx", humans); assertScript("{{#toJson}}.{{/toJson}}", ctx, - equalTo("{\"ctx\":{\"first\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84},\"second\":{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}}}")); + equalTo("{\"ctx\":{\"first\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84},\"second\":" + + "{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}}}")); assertScript("{{#toJson}}ctx{{/toJson}}", ctx, - equalTo("{\"first\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84},\"second\":{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}}")); + equalTo("{\"first\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84},\"second\":" + + "{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}}")); assertScript("{{#toJson}}ctx.first{{/toJson}}", ctx, equalTo("{\"name\":\"John Smith\",\"age\":42,\"height\":1.84}")); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java similarity index 52% rename from modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 90f73da8fb8..c3656029bc4 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -16,137 +16,52 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.script.mustache; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; -import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.template.SearchTemplateAction; -import org.elasticsearch.action.search.template.SearchTemplateRequest; -import org.elasticsearch.action.search.template.SearchTemplateRequestBuilder; -import org.elasticsearch.action.search.template.SearchTemplateResponse; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.script.mustache.TemplateQueryBuilder; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.mustache.MustachePlugin; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; /** * Full integration test of the template query plugin. */ -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) -public class TemplateQueryTests extends ESIntegTestCase { +public class SearchTemplateIT extends ESSingleNodeTestCase { @Override - protected Collection> nodePlugins() { + protected Collection> getPlugins() { return Collections.singleton(MustachePlugin.class); } - @Override - protected Collection> transportClientPlugins() { - return nodePlugins(); - } - @Before public void setup() throws IOException { createIndex("test"); - ensureGreen("test"); - - index("test", "testtype", "1", jsonBuilder().startObject().field("text", "value1").endObject()); - index("test", "testtype", "2", jsonBuilder().startObject().field("text", "value2").endObject()); - refresh(); - } - - @Override - public Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_CONF_SETTING.getKey(), this.getDataPath("config")).build(); - } - - public void testTemplateInBody() throws IOException { - Map vars = new HashMap<>(); - vars.put("template", "all"); - - TemplateQueryBuilder builder = new TemplateQueryBuilder("{\"match_{{template}}\": {}}\"", ScriptType.INLINE,vars); - SearchResponse sr = client().prepareSearch().setQuery(builder) - .execute().actionGet(); - assertHitCount(sr, 2); - } - - public void testTemplateInBodyWithSize() throws IOException { - Map params = new HashMap<>(); - params.put("template", "all"); - SearchResponse sr = client().prepareSearch() - .setSource( - new SearchSourceBuilder().size(0).query( - new TemplateQueryBuilder("{ \"match_{{template}}\": {} }", ScriptType.INLINE, params))) + client().prepareIndex("test", "testtype", "1") + .setSource(jsonBuilder().startObject().field("text", "value1").endObject()) .get(); - assertNoFailures(sr); - assertThat(sr.getHits().hits().length, equalTo(0)); - } - - public void testTemplateWOReplacementInBody() throws IOException { - Map vars = new HashMap<>(); - - TemplateQueryBuilder builder = new TemplateQueryBuilder("{\"match_all\": {}}\"", ScriptType.INLINE, vars); - SearchResponse sr = client().prepareSearch().setQuery(builder) - .execute().actionGet(); - assertHitCount(sr, 2); - } - - public void testTemplateInFile() { - Map vars = new HashMap<>(); - vars.put("template", "all"); - - TemplateQueryBuilder builder = new TemplateQueryBuilder("storedTemplate", ScriptService.ScriptType.FILE, vars); - SearchResponse sr = client().prepareSearch().setQuery(builder) - .execute().actionGet(); - assertHitCount(sr, 2); - } - - public void testRawFSTemplate() throws IOException { - Map params = new HashMap<>(); - params.put("template", "all"); - TemplateQueryBuilder builder = new TemplateQueryBuilder("storedTemplate", ScriptType.FILE, params); - SearchResponse sr = client().prepareSearch().setQuery(builder).get(); - assertHitCount(sr, 2); - } - - public void testSearchRequestTemplateSource() throws Exception { - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("_all"); - - String query = "{ \"inline\" : { \"query\": {\"match_{{template}}\": {} } }, \"params\" : { \"template\":\"all\" } }"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(query)); - request.setRequest(searchRequest); - SearchTemplateResponse response = client().execute(SearchTemplateAction.INSTANCE, request).get(); - assertHitCount(response.getResponse(), 2); + client().prepareIndex("test", "testtype", "2") + .setSource(jsonBuilder().startObject().field("text", "value2").endObject()) + .get(); + client().admin().indices().prepareRefresh().get(); } // Relates to #6318 @@ -173,50 +88,6 @@ public class TemplateQueryTests extends ESIntegTestCase { assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1)); } - public void testThatParametersCanBeSet() throws Exception { - index("test", "type", "1", jsonBuilder().startObject().field("theField", "foo").endObject()); - index("test", "type", "2", jsonBuilder().startObject().field("theField", "foo 2").endObject()); - index("test", "type", "3", jsonBuilder().startObject().field("theField", "foo 3").endObject()); - index("test", "type", "4", jsonBuilder().startObject().field("theField", "foo 4").endObject()); - index("test", "type", "5", jsonBuilder().startObject().field("otherField", "foo").endObject()); - refresh(); - - Map templateParams = new HashMap<>(); - templateParams.put("mySize", "2"); - templateParams.put("myField", "theField"); - templateParams.put("myValue", "foo"); - - SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()) - .setRequest(new SearchRequest("test").types("type")) - .setScript("full-query-template") - .setScriptType(ScriptType.FILE) - .setScriptParams(templateParams) - .get(); - assertHitCount(searchResponse.getResponse(), 4); - // size kicks in here... - assertThat(searchResponse.getResponse().getHits().getHits().length, is(2)); - - templateParams.put("myField", "otherField"); - searchResponse = new SearchTemplateRequestBuilder(client()) - .setRequest(new SearchRequest("test").types("type")) - .setScript("full-query-template") - .setScriptType(ScriptType.FILE) - .setScriptParams(templateParams) - .get(); - assertHitCount(searchResponse.getResponse(), 1); - } - - public void testSearchTemplateQueryFromFile() throws Exception { - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("_all"); - String query = "{" + " \"file\": \"full-query-template\"," + " \"params\":{" + " \"mySize\": 2," - + " \"myField\": \"text\"," + " \"myValue\": \"value1\"" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(query)); - request.setRequest(searchRequest); - SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); - assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1)); - } - /** * Test that template can be expressed as a single escaped string. */ @@ -225,7 +96,7 @@ public class TemplateQueryTests extends ESIntegTestCase { searchRequest.indices("_all"); String query = "{" + " \"inline\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\"," + " \"params\":{" + " \"size\": 1" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(query)); + SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(query)); request.setRequest(searchRequest); SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1)); @@ -241,7 +112,7 @@ public class TemplateQueryTests extends ESIntegTestCase { String templateString = "{" + " \"inline\" : \"{ {{#use_size}} \\\"size\\\": \\\"{{size}}\\\", {{/use_size}} \\\"query\\\":{\\\"match_all\\\":{}}}\"," + " \"params\":{" + " \"size\": 1," + " \"use_size\": true" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(templateString)); + SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(templateString)); request.setRequest(searchRequest); SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1)); @@ -257,7 +128,7 @@ public class TemplateQueryTests extends ESIntegTestCase { String templateString = "{" + " \"inline\" : \"{ \\\"query\\\":{\\\"match_all\\\":{}} {{#use_size}}, \\\"size\\\": \\\"{{size}}\\\" {{/use_size}} }\"," + " \"params\":{" + " \"size\": 1," + " \"use_size\": true" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(templateString)); + SearchTemplateRequest request = RestSearchTemplateAction.parse(new BytesArray(templateString)); request.setRequest(searchRequest); SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1)); @@ -268,39 +139,38 @@ public class TemplateQueryTests extends ESIntegTestCase { .setScriptLang(MustacheScriptEngineService.NAME) .setId("testTemplate") .setSource(new BytesArray("{" + - "\"template\":{" + - " \"query\":{" + - " \"match\":{" + - " \"theField\" : \"{{fieldParam}}\"}" + - " }" + - "}" + - "}"))); + "\"template\":{" + + " \"query\":{" + + " \"match\":{" + + " \"theField\" : \"{{fieldParam}}\"}" + + " }" + + "}" + + "}"))); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(MustacheScriptEngineService.NAME) .setId("testTemplate").setSource(new BytesArray("{" + - "\"template\":{" + - " \"query\":{" + - " \"match\":{" + - " \"theField\" : \"{{fieldParam}}\"}" + - " }" + - "}" + - "}"))); + "\"template\":{" + + " \"query\":{" + + " \"match\":{" + + " \"theField\" : \"{{fieldParam}}\"}" + + " }" + + "}" + + "}"))); GetStoredScriptResponse getResponse = client().admin().cluster() .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get(); assertNotNull(getResponse.getStoredScript()); - List builders = new ArrayList<>(); - - builders.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - builders.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - builders.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - builders.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - builders.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); - - indexRandom(true, builders); + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.get(); + client().admin().indices().prepareRefresh().get(); Map templateParams = new HashMap<>(); templateParams.put("fieldParam", "foo"); @@ -330,20 +200,20 @@ public class TemplateQueryTests extends ESIntegTestCase { .setScriptLang(MustacheScriptEngineService.NAME) .setId("1a") .setSource(new BytesArray("{" + - "\"template\":{"+ - " \"query\":{" + - " \"match\":{" + - " \"theField\" : \"{{fieldParam}}\"}" + - " }" + - "}" + - "}" + "\"template\":{" + + " \"query\":{" + + " \"match\":{" + + " \"theField\" : \"{{fieldParam}}\"}" + + " }" + + "}" + + "}" )) ); assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang(MustacheScriptEngineService.NAME) .setId("2") .setSource(new BytesArray("{" + - "\"template\":{"+ + "\"template\":{" + " \"query\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + @@ -355,20 +225,21 @@ public class TemplateQueryTests extends ESIntegTestCase { .setScriptLang(MustacheScriptEngineService.NAME) .setId("3") .setSource(new BytesArray("{" + - "\"template\":{"+ + "\"template\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}")) ); - List builders = new ArrayList<>(); - builders.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - builders.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - builders.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - builders.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - builders.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); - indexRandom(true, builders); + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.get(); + client().admin().indices().prepareRefresh().get(); Map templateParams = new HashMap<>(); templateParams.put("fieldParam", "foo"); @@ -388,12 +259,12 @@ public class TemplateQueryTests extends ESIntegTestCase { .setScriptParams(templateParams) .get()); - expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()) - .setRequest(new SearchRequest().indices("test").types("type")) - .setScript("/myindex/mustache/1") - .setScriptType(ScriptType.STORED) - .setScriptParams(templateParams) - .get()); + expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()) + .setRequest(new SearchRequest().indices("test").types("type")) + .setScript("/myindex/mustache/1") + .setScriptType(ScriptType.STORED) + .setScriptParams(templateParams) + .get()); templateParams.put("fieldParam", "bar"); searchResponse = new SearchTemplateRequestBuilder(client()) @@ -409,17 +280,6 @@ public class TemplateQueryTests extends ESIntegTestCase { SearchResponse sr = client().prepareSearch().setQuery(builder) .execute().actionGet(); assertHitCount(sr, 1); - - // "{\"template\": {\"id\": \"3\",\"params\" : {\"fieldParam\" : \"foo\"}}}"; - Map params = new HashMap<>(); - params.put("fieldParam", "foo"); - TemplateQueryBuilder templateQuery = new TemplateQueryBuilder("3", ScriptType.STORED, params); - sr = client().prepareSearch().setQuery(templateQuery).get(); - assertHitCount(sr, 4); - - templateQuery = new TemplateQueryBuilder("/mustache/3", ScriptType.STORED, params); - sr = client().prepareSearch().setQuery(templateQuery).get(); - assertHitCount(sr, 4); } // Relates to #10397 @@ -427,13 +287,15 @@ public class TemplateQueryTests extends ESIntegTestCase { createIndex("testindex"); ensureGreen("testindex"); - index("testindex", "test", "1", jsonBuilder().startObject().field("searchtext", "dev1").endObject()); - refresh(); + client().prepareIndex("testindex", "test", "1") + .setSource(jsonBuilder().startObject().field("searchtext", "dev1").endObject()) + .get(); + client().admin().indices().prepareRefresh().get(); int iterations = randomIntBetween(2, 11); for (int i = 1; i < iterations; i++) { assertAcked(client().admin().cluster().preparePutStoredScript() - .setScriptLang(MustacheScriptEngineService.NAME) + .setScriptLang(MustacheScriptEngineService.NAME) .setId("git01") .setSource(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," + "\"type\": \"ooophrase_prefix\"}}}}}"))); @@ -466,26 +328,25 @@ public class TemplateQueryTests extends ESIntegTestCase { } public void testIndexedTemplateWithArray() throws Exception { - String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}"; + String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}"; + assertAcked( + client().admin().cluster().preparePutStoredScript() + .setScriptLang(MustacheScriptEngineService.NAME) + .setId("4") + .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes()) + ); + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); + bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); + bulkRequestBuilder.get(); + client().admin().indices().prepareRefresh().get(); - assertAcked( - client().admin().cluster().preparePutStoredScript() - .setScriptLang(MustacheScriptEngineService.NAME) - .setId("4") - .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes()) - ); - List builders = new ArrayList<>(); - builders.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}")); - builders.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}")); - builders.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}")); - builders.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}")); - builders.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}")); - - indexRandom(true,builders); - - Map arrayTemplateParams = new HashMap<>(); - String[] fieldParams = {"foo","bar"}; - arrayTemplateParams.put("fieldParam", fieldParams); + Map arrayTemplateParams = new HashMap<>(); + String[] fieldParams = {"foo", "bar"}; + arrayTemplateParams.put("fieldParam", fieldParams); SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()) .setRequest(new SearchRequest("test").types("type")) diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java similarity index 98% rename from modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java rename to modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java index bbac6dabcee..b5831a3c5a8 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java @@ -17,12 +17,11 @@ * under the License. */ -package org.elasticsearch.action.search.template; +package org.elasticsearch.script.mustache; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/full-query-template.mustache b/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/full-query-template.mustache deleted file mode 100644 index 519141472f0..00000000000 --- a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/full-query-template.mustache +++ /dev/null @@ -1,6 +0,0 @@ -{ - "query": { - "match": { "{{myField}}" : "{{myValue}}" } - }, - "size" : {{mySize}} -} diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/storedTemplate.mustache b/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/storedTemplate.mustache deleted file mode 100644 index a779da7c467..00000000000 --- a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/storedTemplate.mustache +++ /dev/null @@ -1,3 +0,0 @@ -{ - "match_{{template}}": {} -} diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/action/search/template/simple-msearch-template.json b/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json similarity index 100% rename from modules/lang-mustache/src/test/resources/org/elasticsearch/action/search/template/simple-msearch-template.json rename to modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_render_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml similarity index 72% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_render_search_template.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml index 658d5ed7a90..9a9a1b0c017 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_render_search_template.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml @@ -1,5 +1,5 @@ --- -"Indexed Template validate tests": +"Stored Template validate tests": - do: put_template: @@ -29,6 +29,23 @@ - match: { template_output.query.match.text: "bar" } - match: { template_output.aggs.my_terms.terms.field: "field1" } +--- +"File Template validate tests": + + - do: + render_search_template: + body: { "file": "file_search_template", "params": { "my_value": "foo", "my_field": "field1" } } + + - match: { template_output.query.match.text: "foo" } + - match: { template_output.aggs.my_terms.terms.field: "field1" } + + - do: + render_search_template: + body: { "file": "file_search_template", "params": { "my_value": "bar", "my_field": "my_other_field" } } + + - match: { template_output.query.match.text: "bar" } + - match: { template_output.aggs.my_terms.terms.field: "my_other_field" } + --- "Inline Template validate tests": @@ -51,7 +68,7 @@ render_search_template: body: { "inline": { "query": { "match": { "text": "{{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "bar", "my_field": "field1" } } --- -"Escaped Indexed Template validate tests": +"Escaped Stored Template validate tests": - do: put_template: @@ -102,3 +119,38 @@ catch: /Improperly.closed.variable.in.query-template/ render_search_template: body: { "inline": "{ \"query\": { \"match\": { \"text\": \"{{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "bar", "my_size": 100 } } + +--- +"Indexed Template query tests": + + - do: + index: + index: test + type: testtype + id: 1 + body: { "text": "value1_foo" } + - do: + index: + index: test + type: testtype + id: 2 + body: { "text": "value2_foo value3_foo" } + - do: + indices.refresh: {} + + - do: + put_template: + id: "1" + body: { "template": { "query": { "match" : { "text": "{{my_value}}" } }, "size": "{{my_size}}" } } + - match: { acknowledged: true } + + - do: + search_template: + body: { "id" : "1", "params" : { "my_value" : "value1_foo", "my_size" : 1 } } + - match: { hits.total: 1 } + + - do: + catch: /Unable.to.find.on.disk.file.script.\[simple1\].using.lang.\[mustache\]/ + search_template: + body: { "file" : "simple1"} + diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_search.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_search.yaml deleted file mode 100644 index 4c38dceb14d..00000000000 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_search.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -"Indexed Template query tests": - - - do: - index: - index: test - type: testtype - id: 1 - body: { "text": "value1_foo" } - - do: - index: - index: test - type: testtype - id: 2 - body: { "text": "value2_foo value3_foo" } - - do: - indices.refresh: {} - - - do: - put_template: - id: "1" - body: { "template": { "query": { "match" : { "text": "{{my_value}}" } }, "size": "{{my_size}}" } } - - match: { acknowledged: true } - - - do: - search_template: - body: { "id" : "1", "params" : { "my_value" : "value1_foo", "my_size" : 1 } } - - match: { hits.total: 1 } - - - do: - catch: /Unable.to.find.on.disk.file.script.\[simple1\].using.lang.\[mustache\]/ - search_template: - body: { "file" : "simple1"} - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml new file mode 100644 index 00000000000..c7e4c91b4cc --- /dev/null +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yaml @@ -0,0 +1,108 @@ +--- +"Template search request": + + - do: + index: + index: test + type: testtype + id: 1 + body: { "text": "value1" } + - do: + index: + index: test + type: testtype + id: 2 + body: { "text": "value2" } + - do: + indices.refresh: {} + + - do: + search_template: + body: { "inline" : { "query": { "term": { "text": { "value": "{{template}}" } } } }, "params": { "template": "value1" } } + + - match: { hits.total: 1 } + + - do: + search_template: + body: { "file" : "file_search_template", "params": { "my_value": "value1", "my_field" : "_type" } } + + - match: { hits.total: 1 } + + - do: + put_template: + id: "1" + body: { "template": { "query": { "term": { "text": "{{template}}" } } } } + - match: { acknowledged: true } + + - do: + search_template: + body: { "id" : "1", "params": { "template": "value1" } } + - match: { hits.total: 1 } + + - do: + search_template: + body: { "inline" : { "query": { "match_{{template}}": {} } }, "params" : { "template" : "all" } } + + - match: { hits.total: 2 } + +--- +"Missing template search request": + + - do: + catch: missing + search_template: + body: { "id" : "unknown", "params": { "template": "value1" } } + +--- +"Test that parameters can be used": + + - do: + index: + index: test + type: type + id: 1 + body: { "theField": "foo" } + + - do: + index: + index: test + type: type + id: 2 + body: { "theField": "foo 2" } + + - do: + index: + index: test + type: type + id: 3 + body: { "theField": "foo 3" } + + - do: + index: + index: test + type: type + id: 4 + body: { "theField": "foo 4" } + + - do: + index: + index: test + type: type + id: 5 + body: { "otherField": "foo" } + - do: + indices.refresh: {} + + - do: + search_template: + body: { "file" : "template_1", "params": { "size": "2", "field": "theField", "value": "foo" } } + + - match: { hits.total: 4 } + - length: { hits.hits: 2 } + + - do: + search_template: + body: { "file" : "template_1", "params": { "size": "2", "field": "otherField", "value": "foo" } } + + - match: { hits.total: 1 } + - length: { hits.hits: 1 } diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml deleted file mode 100644 index 8d3154cc8d4..00000000000 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -"Template search request": - - - do: - index: - index: test - type: testtype - id: 1 - body: { "text": "value1" } - - do: - index: - index: test - type: testtype - id: 2 - body: { "text": "value2" } - - do: - indices.refresh: {} - - - do: - search_template: - body: { "inline" : { "query": { "term": { "text": { "value": "{{template}}" } } } }, "params": { "template": "value1" } } - - - match: { hits.total: 1 } - - - do: - search_template: - body: { "inline" : { "query": { "match_{{template}}": {} } }, "params" : { "template" : "all" } } - - - match: { hits.total: 2 } - ---- -"Missing template search request": - - - do: - catch: missing - search_template: - body: { "id" : "unknown", "params": { "template": "value1" } } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_template_query_execution.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml similarity index 62% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_template_query_execution.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml index 62fca6a8ebd..cf3e6883e45 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_template_query_execution.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml @@ -16,12 +16,36 @@ - do: indices.refresh: {} + - do: + put_template: + id: "1" + body: { "template": { "match": { "text": "{{my_value}}" } } } + - match: { acknowledged: true } + - do: search: body: { "query": { "template": { "inline": { "term": { "text": { "value": "{{template}}" } } }, "params": { "template": "value1" } } } } - match: { hits.total: 1 } + - do: + search: + body: { "query": { "template": { "file": "file_query_template", "params": { "my_value": "value1" } } } } + + - match: { hits.total: 1 } + + - do: + search: + body: { "query": { "template": { "id": "1", "params": { "my_value": "value1" } } } } + + - match: { hits.total: 1 } + + - do: + search: + body: { "query": { "template": { "id": "/mustache/1", "params": { "my_value": "value1" } } } } + + - match: { hits.total: 1 } + - do: search: body: { "query": { "template": { "inline": {"match_{{template}}": {}}, "params" : { "template" : "all" } } } } @@ -40,9 +64,13 @@ - match: { hits.total: 2 } + - do: + search: + body: { "query": { "template": { "inline": "{\"match_all\": {}}", "params" : {} } } } + + - match: { hits.total: 2 } + - do: search: body: { "query": { "template": { "inline": "{\"query_string\": { \"query\" : \"{{query}}\" }}", "params" : { "query" : "text:\"value2 value3\"" } } } } - - - match: { hits.total: 1 } diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml deleted file mode 100644 index 24d1ec1efb6..00000000000 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -"Basic multi-search with template query": - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - - - do: - index: - index: test_1 - type: test - id: 2 - body: { foo: baz } - - - do: - index: - index: test_1 - type: test - id: 3 - body: { foo: foo } - - - do: - indices.refresh: {} - - - do: - msearch: - body: - - index: test_1 - - query: { "template": { "inline": { "term": { "foo": { "value": "{{template}}" } } }, "params": { "template": "bar" } } } - - match: { responses.0.hits.total: 1 } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml similarity index 64% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml index ab42133a2b2..a22b101bf67 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yaml @@ -124,3 +124,74 @@ setup: - inline: '{"query": {"match": {"foo": "{{value}}"} } }' params: value: "bar" + +--- +"Basic multi-search using stored template": + + - do: + put_template: + id: stored_template_1 + body: {"template": {"query": {"match": {"{{field}}": "{{value}}" }}}} + - match: { acknowledged: true } + + - do: + msearch_template: + body: + - index: index_* + - id: stored_template_1 + params: + field: "foo" + value: "foo" + - index: _all + - id: stored_template_1 + params: + field: "foo" + value: "bar" + - index: index_2 + - id: stored_template_1 + params: + field: "foo" + value: "foo" + + - match: { responses.0.hits.total: 2 } + - match: { responses.1.hits.total: 1 } + - match: { responses.2.hits.total: 1 } + +--- +"Basic multi-search using file template": + + - do: + render_search_template: + body: { "file": "template_1", "params": { "field": "foo", "value": "bar", "size": 20 } } + + - match: { template_output.query.match.foo.query: "bar" } + - match: { template_output.query.match.foo.operator: "or" } + - match: { template_output.size: 20 } + + - do: + msearch_template: + body: + - index: index_* + - file: template_1 + params: + field: "foo" + value: "foo" + size: 10 + - index: _all + - file: template_1 + params: + field: "foo" + value: "bar" + operator: "and" + size: 50 + - index: index_2 + - file: template_1 + params: + field: "foo" + value: "foo" + size: 0 + + - match: { responses.0.hits.total: 2 } + - match: { responses.1.hits.total: 1 } + - match: { responses.2.hits.total: 1 } + diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml deleted file mode 100644 index ffbe9200375..00000000000 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml +++ /dev/null @@ -1,66 +0,0 @@ ---- -setup: - - - do: - index: - index: index_1 - type: test - id: 1 - body: { foo: bar } - - - do: - index: - index: index_1 - type: test - id: 2 - body: { foo: baz } - - - do: - index: - index: index_1 - type: test - id: 3 - body: { foo: foo } - - - do: - index: - index: index_2 - type: test - id: 1 - body: { foo: foo } - - - do: - indices.refresh: {} - ---- -"Basic multi-search using stored template": - - - do: - put_template: - id: stored_template_1 - body: {"template": {"query": {"match": {"{{field}}": "{{value}}" }}}} - - match: { acknowledged: true } - - - do: - msearch_template: - body: - - index: index_* - - id: stored_template_1 - params: - field: "foo" - value: "foo" - - index: _all - - id: stored_template_1 - params: - field: "foo" - value: "bar" - - index: index_2 - - id: stored_template_1 - params: - field: "foo" - value: "foo" - - - match: { responses.0.hits.total: 2 } - - match: { responses.1.hits.total: 1 } - - match: { responses.2.hits.total: 1 } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml deleted file mode 100644 index a5d11898baf..00000000000 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml +++ /dev/null @@ -1,72 +0,0 @@ ---- -setup: - - - do: - index: - index: index_1 - type: test - id: 1 - body: { foo: bar } - - - do: - index: - index: index_1 - type: test - id: 2 - body: { foo: baz } - - - do: - index: - index: index_1 - type: test - id: 3 - body: { foo: foo } - - - do: - index: - index: index_2 - type: test - id: 1 - body: { foo: foo } - - - do: - indices.refresh: {} - ---- -"Basic multi-search using file template": - - - do: - render_search_template: - body: { "file": "template_1", "params": { "field": "foo", "value": "bar", "size": 20 } } - - - match: { template_output.query.match.foo.query: "bar" } - - match: { template_output.query.match.foo.operator: "or" } - - match: { template_output.size: 20 } - - - do: - msearch_template: - body: - - index: index_* - - file: template_1 - params: - field: "foo" - value: "foo" - size: 10 - - index: _all - - file: template_1 - params: - field: "foo" - value: "bar" - operator: "and" - size: 50 - - index: index_2 - - file: template_1 - params: - field: "foo" - value: "foo" - size: 0 - - - match: { responses.0.hits.total: 2 } - - match: { responses.1.hits.total: 1 } - - match: { responses.2.hits.total: 1 } - diff --git a/modules/lang-mustache/src/test/resources/templates/file_query_template.mustache b/modules/lang-mustache/src/test/resources/templates/file_query_template.mustache new file mode 100644 index 00000000000..f083a132133 --- /dev/null +++ b/modules/lang-mustache/src/test/resources/templates/file_query_template.mustache @@ -0,0 +1,5 @@ +{ + "match": { + "text": "{{my_value}}" + } +} diff --git a/modules/lang-mustache/src/test/resources/templates/file_search_template.mustache b/modules/lang-mustache/src/test/resources/templates/file_search_template.mustache new file mode 100644 index 00000000000..beaf32ac0f1 --- /dev/null +++ b/modules/lang-mustache/src/test/resources/templates/file_search_template.mustache @@ -0,0 +1,12 @@ +{ + "query": { + "match": { + "text": "{{my_value}}" + } + }, + "aggs": { + "my_terms": { + "terms": { "field": "{{my_field}}" } + } + } +} diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache b/modules/lang-mustache/src/test/resources/templates/template_1.mustache similarity index 99% rename from modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache rename to modules/lang-mustache/src/test/resources/templates/template_1.mustache index 4b1c02f3b24..29174be2ff0 100644 --- a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache +++ b/modules/lang-mustache/src/test/resources/templates/template_1.mustache @@ -8,4 +8,4 @@ } }, "size": {{size}} -} \ No newline at end of file +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessRestIT.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessRestIT.java index dbb596c275c..353aa13f839 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessRestIT.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessRestIT.java @@ -21,14 +21,14 @@ package org.elasticsearch.painless; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; /** Runs yaml rest tests */ -public class PainlessRestIT extends ESRestTestCase { +public class PainlessRestIT extends ESClientYamlSuiteTestCase { public PainlessRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -36,7 +36,7 @@ public class PainlessRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java index 9f4eff64995..f4ec1b5215e 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java @@ -21,19 +21,19 @@ package org.elasticsearch.percolator; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class PercolatorRestIT extends ESRestTestCase { +public class PercolatorRestIT extends ESClientYamlSuiteTestCase { public PercolatorRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 0d02d1d9474..fa4c44d9224 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -35,16 +35,11 @@ run { setting 'reindex.remote.whitelist', 'myself' } - dependencies { compile "org.elasticsearch.client:rest:${version}" - // dependencies of the rest client - compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" - compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" - compile "commons-codec:commons-codec:${versions.commonscodec}" - compile "commons-logging:commons-logging:${versions.commonslogging}" // for http - testing reindex from remote testCompile project(path: ':modules:transport-netty3', configuration: 'runtime') + testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') } dependencyLicenses { diff --git a/modules/reindex/licenses/commons-codec-1.10.jar.sha1 b/modules/reindex/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0..00000000000 --- a/modules/reindex/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/modules/reindex/licenses/commons-codec-NOTICE.txt b/modules/reindex/licenses/commons-codec-NOTICE.txt deleted file mode 100644 index 56916449bbe..00000000000 --- a/modules/reindex/licenses/commons-codec-NOTICE.txt +++ /dev/null @@ -1,17 +0,0 @@ -Apache Commons Codec -Copyright 2002-2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. diff --git a/modules/reindex/licenses/commons-logging-1.1.3.jar.sha1 b/modules/reindex/licenses/commons-logging-1.1.3.jar.sha1 deleted file mode 100644 index c8756c43832..00000000000 --- a/modules/reindex/licenses/commons-logging-1.1.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f diff --git a/modules/reindex/licenses/commons-logging-NOTICE.txt b/modules/reindex/licenses/commons-logging-NOTICE.txt deleted file mode 100644 index 72eb32a9024..00000000000 --- a/modules/reindex/licenses/commons-logging-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons CLI -Copyright 2001-2009 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/reindex/licenses/httpclient-4.5.2.jar.sha1 b/modules/reindex/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09f..00000000000 --- a/modules/reindex/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/modules/reindex/licenses/httpclient-LICENSE.txt b/modules/reindex/licenses/httpclient-LICENSE.txt deleted file mode 100644 index 32f01eda18f..00000000000 --- a/modules/reindex/licenses/httpclient-LICENSE.txt +++ /dev/null @@ -1,558 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -========================================================================= - -This project includes Public Suffix List copied from - -licensed under the terms of the Mozilla Public License, v. 2.0 - -Full license text: - -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. diff --git a/modules/reindex/licenses/httpclient-NOTICE.txt b/modules/reindex/licenses/httpclient-NOTICE.txt deleted file mode 100644 index 4f6058178b2..00000000000 --- a/modules/reindex/licenses/httpclient-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache HttpComponents Client -Copyright 1999-2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/reindex/licenses/httpcore-4.4.4.jar.sha1 b/modules/reindex/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/modules/reindex/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/modules/reindex/licenses/httpcore-LICENSE.txt b/modules/reindex/licenses/httpcore-LICENSE.txt deleted file mode 100644 index 72819a9f06f..00000000000 --- a/modules/reindex/licenses/httpcore-LICENSE.txt +++ /dev/null @@ -1,241 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -========================================================================= - -This project contains annotations in the package org.apache.http.annotation -which are derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. -See http://www.jcip.net and the Creative Commons Attribution License -(http://creativecommons.org/licenses/by/2.5) -Full text: http://creativecommons.org/licenses/by/2.5/legalcode - -License - -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - - "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License. - "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License. - "Licensor" means the individual or entity that offers the Work under the terms of this License. - "Original Author" means the individual or entity who created the Work. - "Work" means the copyrightable work of authorship offered under the terms of this License. - "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. - -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: - - to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works; - to create and reproduce Derivative Works; - to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works; - to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works. - - For the avoidance of doubt, where the work is a musical composition: - Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work. - Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions). - Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions). - -The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: - - You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested. - If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit. - -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. Termination - - This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. - Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. - -8. Miscellaneous - - Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. - Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. - If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. - No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. - This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. diff --git a/modules/reindex/licenses/httpcore-NOTICE.txt b/modules/reindex/licenses/httpcore-NOTICE.txt deleted file mode 100644 index c0be50a505e..00000000000 --- a/modules/reindex/licenses/httpcore-NOTICE.txt +++ /dev/null @@ -1,8 +0,0 @@ -Apache HttpComponents Core -Copyright 2005-2014 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 3e6f806d293..2238bec433b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -48,7 +48,6 @@ import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.mapper.internal.VersionFieldMapper; import org.elasticsearch.index.reindex.remote.RemoteInfo; import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource; -import org.elasticsearch.node.service.NodeService; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; @@ -185,12 +184,10 @@ public class TransportReindexAction extends HandledTransportAction fail, AsyncClient client, BytesReference query, SearchRequest searchRequest) { + Consumer fail, RestClient client, BytesReference query, SearchRequest searchRequest) { super(logger, backoffPolicy, threadPool, countSearchRetry, fail); this.query = query; this.searchRequest = searchRequest; @@ -99,7 +97,7 @@ public class RemoteScrollableHitSource extends ScrollableHitSource { } - void onStartResponse(Consumer onResponse, Response response) { + private void onStartResponse(Consumer onResponse, Response response) { if (Strings.hasLength(response.getScrollId()) && response.getHits().isEmpty()) { logger.debug("First response looks like a scan response. Jumping right to the second. scroll=[{}]", response.getScrollId()); doStartNextScroll(response.getScrollId(), timeValueMillis(0), onResponse); @@ -119,15 +117,10 @@ public class RemoteScrollableHitSource extends ScrollableHitSource { // Need to throw out response.... client.performRequest("DELETE", scrollPath(), emptyMap(), scrollEntity(scrollId), new ResponseListener() { @Override - public void onResponse(InputStream response) { + public void onSuccess(org.elasticsearch.client.Response response) { logger.debug("Successfully cleared [{}]", scrollId); } - @Override - public void onRetryableFailure(Exception t) { - onFailure(t); - } - @Override public void onFailure(Exception t) { logger.warn("Failed to clear scroll [{}]", t, scrollId); @@ -135,7 +128,7 @@ public class RemoteScrollableHitSource extends ScrollableHitSource { }); } - void execute(String method, String uri, Map params, HttpEntity entity, + private void execute(String method, String uri, Map params, HttpEntity entity, BiFunction parser, Consumer listener) { class RetryHelper extends AbstractRunnable { private final Iterator retries = backoffPolicy.iterator(); @@ -144,34 +137,43 @@ public class RemoteScrollableHitSource extends ScrollableHitSource { protected void doRun() throws Exception { client.performRequest(method, uri, params, entity, new ResponseListener() { @Override - public void onResponse(InputStream content) { - T response; + public void onSuccess(org.elasticsearch.client.Response response) { + T parsedResponse; try { - XContent xContent = XContentFactory.xContentType(content).xContent(); - try(XContentParser xContentParser = xContent.createParser(content)) { - response = parser.apply(xContentParser, () -> ParseFieldMatcher.STRICT); + HttpEntity responseEntity = response.getEntity(); + InputStream content = responseEntity.getContent(); + XContentType xContentType = null; + if (responseEntity.getContentType() != null) { + xContentType = XContentType.fromMediaTypeOrFormat(responseEntity.getContentType().getValue()); + } + if (xContentType == null) { + //auto-detect as a fallback + xContentType = XContentFactory.xContentType(content); + } + try(XContentParser xContentParser = xContentType.xContent().createParser(content)) { + parsedResponse = parser.apply(xContentParser, () -> ParseFieldMatcher.STRICT); } } catch (IOException e) { throw new ElasticsearchException("Error deserializing response", e); } - listener.accept(response); + listener.accept(parsedResponse); } @Override public void onFailure(Exception e) { - fail.accept(e); - } - - @Override - public void onRetryableFailure(Exception t) { - if (retries.hasNext()) { - TimeValue delay = retries.next(); - logger.trace("retrying rejected search after [{}]", t, delay); - countSearchRetry.run(); - threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this); - } else { - fail.accept(t); + if (e instanceof ResponseException) { + ResponseException re = (ResponseException) e; + if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) { + if (retries.hasNext()) { + TimeValue delay = retries.next(); + logger.trace("retrying rejected search after [{}]", e, delay); + countSearchRetry.run(); + threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this); + return; + } + } } + fail.accept(e); } }); } @@ -183,60 +185,4 @@ public class RemoteScrollableHitSource extends ScrollableHitSource { } new RetryHelper().run(); } - - public interface AsyncClient extends Closeable { - void performRequest(String method, String uri, Map params, HttpEntity entity, ResponseListener listener); - } - - public interface ResponseListener extends ActionListener { - void onRetryableFailure(Exception t); - } - - public static class AsynchronizingRestClient implements AsyncClient { - private final ThreadPool threadPool; - private final RestClient restClient; - - public AsynchronizingRestClient(ThreadPool threadPool, RestClient restClient) { - this.threadPool = threadPool; - this.restClient = restClient; - } - - @Override - public void performRequest(String method, String uri, Map params, HttpEntity entity, - ResponseListener listener) { - /* - * We use the generic thread pool here because this client is blocking the generic thread pool is sized appropriately for some - * of the threads on it to be blocked, waiting on IO. It'd be a disaster if this ran on the listener thread pool, eating - * valuable threads needed to handle responses. Most other thread pool would probably not mind running this either, but the - * generic thread pool is the "most right" place for it to run. We could make our own thread pool for this but the generic - * thread pool already has plenty of capacity. - */ - threadPool.generic().execute(new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - try (org.elasticsearch.client.Response response = restClient.performRequest(method, uri, params, entity)) { - InputStream markSupportedInputStream = new BufferedInputStream(response.getEntity().getContent()); - listener.onResponse(markSupportedInputStream); - } - } - - @Override - public void onFailure(Exception t) { - if (t instanceof ResponseException) { - ResponseException re = (ResponseException) t; - if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) { - listener.onRetryableFailure(t); - return; - } - } - listener.onFailure(t); - } - }); - } - - @Override - public void close() throws IOException { - restClient.close(); - } - } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestIT.java index d8718c5b493..c67908f631d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestIT.java @@ -22,19 +22,19 @@ package org.elasticsearch.index.reindex; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class ReindexRestIT extends ESRestTestCase { +public class ReindexRestIT extends ESClientYamlSuiteTestCase { public ReindexRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java index fd251d96a1f..2be27a1a1ad 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Netty3Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.junit.After; import org.junit.Before; @@ -56,17 +57,47 @@ import static org.hamcrest.Matchers.hasSize; * tests won't verify that. */ public class RetryTests extends ESSingleNodeTestCase { + private static final int DOC_COUNT = 20; private List blockedExecutors = new ArrayList<>(); + private boolean useNetty4; + + @Before + public void setUp() throws Exception { + super.setUp(); + useNetty4 = randomBoolean(); + createIndex("source"); + // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. + BulkRequestBuilder bulk = client().prepareBulk(); + for (int i = 0; i < DOC_COUNT; i++) { + bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i)); + } + Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff()); + BulkResponse response = retry.withSyncBackoff(client(), bulk.request()); + assertFalse(response.buildFailureMessage(), response.hasFailures()); + client().admin().indices().prepareRefresh("source").get(); + } + + @After + public void forceUnblockAllExecutors() { + for (CyclicBarrier barrier: blockedExecutors) { + barrier.reset(); + } + } + @Override protected Collection> getPlugins() { - return pluginList(ReindexPlugin.class, Netty3Plugin.class, BogusPlugin.class); // we need netty here to http communication + return pluginList( + ReindexPlugin.class, + Netty3Plugin.class, + Netty4Plugin.class, + BogusPlugin.class); } public static final class BogusPlugin extends Plugin { - // se Netty3Plugin.... this runs without the permission from the netty3 module so it will fail since reindex can't set the property + // this runs without the permission from the netty module so it will fail since reindex can't set the property // to make it still work we disable that check but need to register the setting first private static final Setting ASSERT_NETTY_BUGLEVEL = Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope); @@ -94,30 +125,13 @@ public class RetryTests extends ESSingleNodeTestCase { settings.put(NetworkModule.HTTP_ENABLED.getKey(), true); // Whitelist reindexing from the http host we're going to use settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "myself"); + if (useNetty4) { + settings.put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME); + settings.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + } return settings.build(); } - @Before - public void setupSourceIndex() throws Exception { - createIndex("source"); - // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools. - BulkRequestBuilder bulk = client().prepareBulk(); - for (int i = 0; i < DOC_COUNT; i++) { - bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i)); - } - Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff()); - BulkResponse response = retry.withSyncBackoff(client(), bulk.request()); - assertFalse(response.buildFailureMessage(), response.hasFailures()); - client().admin().indices().prepareRefresh("source").get(); - } - - @After - public void forceUnblockAllExecutors() { - for (CyclicBarrier barrier: blockedExecutors) { - barrier.reset(); - } - } - public void testReindex() throws Exception { testCase(ReindexAction.NAME, ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest"), matcher().created(DOC_COUNT)); @@ -222,4 +236,5 @@ public class RetryTests extends ESSingleNodeTestCase { assertThat(response.getTasks(), hasSize(1)); return (BulkByScrollTask.Status) response.getTasks().get(0).getStatus(); } + } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java index f8f3e82b4bb..6af4dab9405 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java @@ -19,31 +19,45 @@ package org.elasticsearch.index.reindex.remote; -import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.StatusLine; +import org.apache.http.concurrent.FutureCallback; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; +import org.apache.http.nio.protocol.HttpAsyncRequestProducer; +import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import org.elasticsearch.Version; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.reindex.ScrollableHitSource.Response; -import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource.ResponseListener; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; -import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; @@ -53,6 +67,9 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class RemoteScrollableHitSourceTests extends ESTestCase { private final String FAKE_SCROLL_ID = "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll"; @@ -68,7 +85,7 @@ public class RemoteScrollableHitSourceTests extends ESTestCase { threadPool = new TestThreadPool(getTestName()) { @Override public Executor executor(String name) { - return r -> r.run(); + return Runnable::run; } @Override @@ -307,6 +324,7 @@ public class RemoteScrollableHitSourceTests extends ESTestCase { * Creates a hit source that doesn't make the remote request and instead returns data from some files. Also requests are always returned * synchronously rather than asynchronously. */ + @SuppressWarnings("unchecked") private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteVersion, String... paths) throws Exception { URL[] resources = new URL[paths.length]; for (int i = 0; i < paths.length; i++) { @@ -315,35 +333,49 @@ public class RemoteScrollableHitSourceTests extends ESTestCase { throw new IllegalArgumentException("Couldn't find [" + paths[i] + "]"); } } - RemoteScrollableHitSource.AsyncClient client = new RemoteScrollableHitSource.AsyncClient() { + + CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class); + when(httpClient.execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class), + any(FutureCallback.class))).thenAnswer(new Answer>() { + int responseCount = 0; - @Override - public void performRequest(String method, String uri, Map params, HttpEntity entity, - ResponseListener listener) { - try { - URL resource = resources[responseCount]; - String path = paths[responseCount++]; - InputStream stream = resource.openStream(); - if (path.startsWith("fail:")) { - String body = Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)); - if (path.equals("fail:rejection.json")) { - listener.onRetryableFailure(new RuntimeException(body)); - } else { - listener.onFailure(new RuntimeException(body)); - } - } else { - listener.onResponse(stream); - } - } catch (IOException e) { - listener.onFailure(e); - } - } @Override - public void close() throws IOException { + public Future answer(InvocationOnMock invocationOnMock) throws Throwable { + HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0]; + @SuppressWarnings("unchecked") + FutureCallback futureCallback = (FutureCallback) invocationOnMock.getArguments()[2]; + HttpEntityEnclosingRequest request = (HttpEntityEnclosingRequest)requestProducer.generateRequest(); + URL resource = resources[responseCount]; + String path = paths[responseCount++]; + ProtocolVersion protocolVersion = new ProtocolVersion("http", 1, 1); + if (path.startsWith("fail:")) { + String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); + if (path.equals("fail:rejection.json")) { + StatusLine statusLine = new BasicStatusLine(protocolVersion, RestStatus.TOO_MANY_REQUESTS.getStatus(), ""); + BasicHttpResponse httpResponse = new BasicHttpResponse(statusLine); + futureCallback.completed(httpResponse); + } else { + futureCallback.failed(new RuntimeException(body)); + } + } else { + StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, ""); + HttpResponse httpResponse = new BasicHttpResponse(statusLine); + httpResponse.setEntity(new InputStreamEntity(resource.openStream(), + randomBoolean() ? ContentType.APPLICATION_JSON : null)); + futureCallback.completed(httpResponse); + } + return null; } - }; - TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client) { + }); + + HttpAsyncClientBuilder clientBuilder = mock(HttpAsyncClientBuilder.class); + when(clientBuilder.build()).thenReturn(httpClient); + + RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)) + .setHttpClientConfigCallback(httpClientBuilder -> clientBuilder).build(); + + TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(restClient) { @Override void lookupRemoteVersion(Consumer onVersion) { if (mockRemoteVersion) { @@ -372,7 +404,7 @@ public class RemoteScrollableHitSourceTests extends ESTestCase { } private class TestRemoteScrollableHitSource extends RemoteScrollableHitSource { - public TestRemoteScrollableHitSource(RemoteScrollableHitSource.AsyncClient client) { + TestRemoteScrollableHitSource(RestClient client) { super(RemoteScrollableHitSourceTests.this.logger, backoff(), RemoteScrollableHitSourceTests.this.threadPool, RemoteScrollableHitSourceTests.this::countRetry, RemoteScrollableHitSourceTests.this::failRequest, client, new BytesArray("{}"), RemoteScrollableHitSourceTests.this.searchRequest); diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java index 39a5c94b239..abf23563b8a 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequest.java @@ -26,40 +26,27 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.support.RestUtils; import org.jboss.netty.channel.Channel; import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; import java.net.SocketAddress; import java.util.HashMap; import java.util.Map; -/** - * - */ public class Netty3HttpRequest extends RestRequest { - private final org.jboss.netty.handler.codec.http.HttpRequest request; + private final HttpRequest request; private final Channel channel; - private final Map params; - private final String rawPath; private final BytesReference content; - public Netty3HttpRequest(org.jboss.netty.handler.codec.http.HttpRequest request, Channel channel) { + public Netty3HttpRequest(HttpRequest request, Channel channel) { + super(request.getUri()); this.request = request; this.channel = channel; - this.params = new HashMap<>(); if (request.getContent().readable()) { this.content = Netty3Utils.toBytesReference(request.getContent()); } else { this.content = BytesArray.EMPTY; } - - String uri = request.getUri(); - int pathEndPos = uri.indexOf('?'); - if (pathEndPos < 0) { - this.rawPath = uri; - } else { - this.rawPath = uri.substring(0, pathEndPos); - RestUtils.decodeQueryString(uri, pathEndPos + 1, params); - } } public org.jboss.netty.handler.codec.http.HttpRequest request() { @@ -97,16 +84,6 @@ public class Netty3HttpRequest extends RestRequest { return request.getUri(); } - @Override - public String rawPath() { - return rawPath; - } - - @Override - public Map params() { - return params; - } - @Override public boolean hasContent() { return content.length() > 0; @@ -153,22 +130,4 @@ public class Netty3HttpRequest extends RestRequest { return request.headers().entries(); } - @Override - public boolean hasParam(String key) { - return params.containsKey(key); - } - - @Override - public String param(String key) { - return params.get(key); - } - - @Override - public String param(String key, String defaultValue) { - String value = params.get(key); - if (value == null) { - return defaultValue; - } - return value; - } } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java index 968eb6a24ab..829a2d9aa06 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpRequestHandler.java @@ -54,7 +54,6 @@ public class Netty3HttpRequestHandler extends SimpleChannelUpstreamHandler { request = (HttpRequest) e.getMessage(); } - threadContext.copyHeaders(request.headers()); // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally // when reading, or using a cumalation buffer Netty3HttpRequest httpRequest = new Netty3HttpRequest(request, e.getChannel()); diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java index c480155dceb..25884d14b70 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/Netty3HttpServerTransport.java @@ -81,9 +81,11 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.elasticsearch.common.settings.Setting.boolSetting; import static org.elasticsearch.common.settings.Setting.byteSizeSetting; @@ -118,29 +120,29 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem public static Setting SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY = Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), - Property.NodeScope); + Property.NodeScope, Property.Shared); public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = - Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope); + Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count", (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), - (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope); + (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_NO_DELAY = - boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope); + boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_KEEP_ALIVE = - boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope); + boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_BLOCKING_SERVER = - boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope); + boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_REUSE_ADDRESS = - boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope); + boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting("http.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE, - Property.NodeScope); + Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE, - Property.NodeScope); + Property.NodeScope, Property.Shared); public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting("transport.netty.receive_predictor_size", settings -> { @@ -390,14 +392,10 @@ public class Netty3HttpServerTransport extends AbstractLifecycleComponent implem if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) { builder.allowCredentials(); } - String[] strMethods = settings.getAsArray(SETTING_CORS_ALLOW_METHODS.getKey()); - HttpMethod[] methods = Arrays.asList(strMethods) - .stream() - .map(HttpMethod::valueOf) - .toArray(size -> new HttpMethod[size]); - return builder.allowedRequestMethods(methods) + Set strMethods = Strings.splitStringByCommaToSet(SETTING_CORS_ALLOW_METHODS.get(settings)); + return builder.allowedRequestMethods(strMethods.stream().map(HttpMethod::valueOf).collect(Collectors.toSet())) .maxAge(SETTING_CORS_MAX_AGE.get(settings)) - .allowedRequestHeaders(settings.getAsArray(SETTING_CORS_ALLOW_HEADERS.getKey())) + .allowedRequestHeaders(Strings.splitStringByCommaToSet(SETTING_CORS_ALLOW_HEADERS.get(settings))) .shortCircuit() .build(); } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsConfigBuilder.java b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsConfigBuilder.java index 947ec86b161..e7b94898b14 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsConfigBuilder.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/http/netty3/cors/Netty3CorsConfigBuilder.java @@ -193,8 +193,8 @@ public final class Netty3CorsConfigBuilder { * @param methods the {@link HttpMethod}s that should be allowed. * @return {@link Netty3CorsConfigBuilder} to support method chaining. */ - public Netty3CorsConfigBuilder allowedRequestMethods(final HttpMethod... methods) { - requestMethods.addAll(Arrays.asList(methods)); + public Netty3CorsConfigBuilder allowedRequestMethods(final Set methods) { + requestMethods.addAll(methods); return this; } @@ -214,8 +214,8 @@ public final class Netty3CorsConfigBuilder { * @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header. * @return {@link Netty3CorsConfigBuilder} to support method chaining. */ - public Netty3CorsConfigBuilder allowedRequestHeaders(final String... headers) { - requestHeaders.addAll(Arrays.asList(headers)); + public Netty3CorsConfigBuilder allowedRequestHeaders(final Set headers) { + requestHeaders.addAll(headers); return this; } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java index ff4dc27e2c8..e2ac10873cd 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3MessageChannelHandler.java @@ -50,7 +50,7 @@ class Netty3MessageChannelHandler extends SimpleChannelUpstreamHandler { @Override public void writeComplete(ChannelHandlerContext ctx, WriteCompletionEvent e) throws Exception { - transportServiceAdapter.sent(e.getWrittenAmount()); + transportServiceAdapter.addBytesSent(e.getWrittenAmount()); super.writeComplete(ctx, e); } diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java index 5ca2252cf2c..f5f32044eb7 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Transport.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.Booleans; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.network.NetworkService.TcpSettings; @@ -45,6 +46,7 @@ import org.elasticsearch.transport.TransportServiceAdapter; import org.elasticsearch.transport.TransportSettings; import org.jboss.netty.bootstrap.ClientBootstrap; import org.jboss.netty.bootstrap.ServerBootstrap; +import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.channel.AdaptiveReceiveBufferSizePredictorFactory; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFuture; @@ -93,31 +95,36 @@ public class Netty3Transport extends TcpTransport { public static final Setting WORKER_COUNT = new Setting<>("transport.netty.worker_count", (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), - (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope); + (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope, Property.Shared); public static final Setting NETTY_MAX_CUMULATION_BUFFER_CAPACITY = - Setting.byteSizeSetting("transport.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), Property.NodeScope); + Setting.byteSizeSetting( + "transport.netty.max_cumulation_buffer_capacity", + new ByteSizeValue(-1), + Property.NodeScope, + Property.Shared); public static final Setting NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = - Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope); + Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope, Property.Shared); // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one public static final Setting NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting( - "transport.netty.receive_predictor_size", - settings -> { - long defaultReceiverPredictor = 512 * 1024; - if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { - // we can guess a better default... - long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings)); - defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); - } - return new ByteSizeValue(defaultReceiverPredictor).toString(); - }, Property.NodeScope); + "transport.netty.receive_predictor_size", + settings -> { + long defaultReceiverPredictor = 512 * 1024; + if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { + // we can guess a better default... + long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings)); + defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); + } + return new ByteSizeValue(defaultReceiverPredictor).toString(); + }, Property.NodeScope, + Property.Shared); public static final Setting NETTY_RECEIVE_PREDICTOR_MIN = - byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); public static final Setting NETTY_RECEIVE_PREDICTOR_MAX = - byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); public static final Setting NETTY_BOSS_COUNT = - intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope); + intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope, Property.Shared); protected final ByteSizeValue maxCumulationBufferCapacity; @@ -557,4 +564,5 @@ public class Netty3Transport extends TcpTransport { } }); } + } diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java b/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java index a7f8f254fcc..3c90bc4ccea 100644 --- a/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java +++ b/modules/transport-netty3/src/test/java/org/elasticsearch/ESNetty3IntegTestCase.java @@ -47,15 +47,15 @@ public abstract class ESNetty3IntegTestCase extends ESIntegTestCase { if (randomBoolean()) { builder.put(Netty3Transport.WORKER_COUNT.getKey(), random().nextInt(3) + 1); } - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3"); - builder.put(NetworkModule.HTTP_TYPE_KEY, "netty3"); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME); + builder.put(NetworkModule.HTTP_TYPE_KEY, Netty3Plugin.NETTY_HTTP_TRANSPORT_NAME); return builder.build(); } @Override protected Settings transportClientSettings() { Settings.Builder builder = Settings.builder().put(super.transportClientSettings()); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3"); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME); return builder.build(); } diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java index da7e320a557..901d517bf95 100644 --- a/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java +++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3HttpServerTransportTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.http.netty3.cors.Netty3CorsConfig; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; @@ -86,4 +87,19 @@ public class Netty3HttpServerTransportTests extends ESTestCase { assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::getName).collect(Collectors.toSet()), equalTo(methods)); transport.close(); } + + public void testCorsConfigDefaults() { + final Set headers = Sets.newHashSet("X-Requested-With", "Content-Type", "Content-Length"); + final Set methods = Sets.newHashSet("OPTIONS", "HEAD", "GET", "POST", "PUT", "DELETE"); + final Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*") + .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true) + .build(); + final Netty3HttpServerTransport transport = new Netty3HttpServerTransport(settings, networkService, bigArrays, threadPool); + final Netty3CorsConfig corsConfig = transport.getCorsConfig(); + assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers)); + assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::getName).collect(Collectors.toSet()), equalTo(methods)); + transport.close(); + } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3RestIT.java similarity index 54% rename from client/rest/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java rename to modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3RestIT.java index dd866bac541..9aa57bc312f 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java +++ b/modules/transport-netty3/src/test/java/org/elasticsearch/http/netty3/Netty3RestIT.java @@ -17,26 +17,25 @@ * under the License. */ -package org.elasticsearch.client; +package org.elasticsearch.http.netty3; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.message.BasicHttpResponse; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -/** - * Simple {@link CloseableHttpResponse} impl needed to easily create http responses that are closeable given that - * org.apache.http.impl.execchain.HttpResponseProxy is not public. - */ -class CloseableBasicHttpResponse extends BasicHttpResponse implements CloseableHttpResponse { +public class Netty3RestIT extends ESClientYamlSuiteTestCase { - public CloseableBasicHttpResponse(StatusLine statusline) { - super(statusline); + public Netty3RestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); } - @Override - public void close() throws IOException { - //nothing to close + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESClientYamlSuiteTestCase.createParameters(0, 1); } -} \ No newline at end of file +} diff --git a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java index 8b9b2d397db..d25951e254c 100644 --- a/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java +++ b/modules/transport-netty3/src/test/java/org/elasticsearch/transport/netty3/Netty3TransportMultiPortIntegrationIT.java @@ -73,7 +73,7 @@ public class Netty3TransportMultiPortIntegrationIT extends ESNetty3IntegTestCase public void testThatTransportClientCanConnect() throws Exception { Settings settings = Settings.builder() .put("cluster.name", internalCluster().getClusterName()) - .put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3") + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty3Plugin.NETTY_TRANSPORT_NAME) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); try (TransportClient transportClient = new MockTransportClient(settings, Netty3Plugin.class)) { diff --git a/modules/transport-netty3/src/test/resources/rest-api-spec/test/10_basic.yaml b/modules/transport-netty3/src/test/resources/rest-api-spec/test/10_basic.yaml index eaf51de4484..e25074fb90c 100644 --- a/modules/transport-netty3/src/test/resources/rest-api-spec/test/10_basic.yaml +++ b/modules/transport-netty3/src/test/resources/rest-api-spec/test/10_basic.yaml @@ -10,4 +10,4 @@ - do: nodes.info: {} - - match: { nodes.$master.modules.0.name: transport-netty3 } \ No newline at end of file + - match: { nodes.$master.modules.0.name: transport-netty3 } diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle new file mode 100644 index 00000000000..82d98a981ab --- /dev/null +++ b/modules/transport-netty4/build.gradle @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + TODOs: + * fix permissions such that only netty4 can open sockets etc? + * fix the hack in the build framework that copies transport-netty4 into the integ test cluster + * maybe figure out a way to run all tests from core with netty4/network? + */ +esplugin { + description 'Netty 4 based transport implementation' + classname 'org.elasticsearch.transport.Netty4Plugin' + hasClientJar = true +} + +compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" + +dependencies { + // network stack + compile "io.netty:netty-buffer:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-codec:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-codec-http:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-common:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-handler:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-resolver:4.1.4.Final-elastic-SNAPSHOT" + compile "io.netty:netty-transport:4.1.4.Final-elastic-SNAPSHOT" +} + +integTest { + includePackaged = true + cluster { + setting 'http.type', 'netty4' + setting 'transport.type', 'netty4' + numNodes = 2 + } +} + +thirdPartyAudit.excludes = [ + // classes are missing + + // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty) + 'com.google.protobuf.ExtensionRegistry', + 'com.google.protobuf.MessageLite$Builder', + 'com.google.protobuf.MessageLite', + 'com.google.protobuf.Parser', + + // from io.netty.logging.CommonsLoggerFactory (netty) + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + + // from io.netty.handler.ssl.OpenSslEngine (netty) + 'org.apache.tomcat.jni.Buffer', + 'org.apache.tomcat.jni.Library', + 'org.apache.tomcat.jni.Pool', + 'org.apache.tomcat.jni.SSL', + 'org.apache.tomcat.jni.SSLContext', + + // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty) + 'org.bouncycastle.asn1.x500.X500Name', + 'org.bouncycastle.cert.X509v3CertificateBuilder', + 'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter', + 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', + 'org.bouncycastle.jce.provider.BouncyCastleProvider', + 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + + // from io.netty.handler.ssl.JettyNpnSslEngine (netty) + 'org.eclipse.jetty.npn.NextProtoNego$ClientProvider', + 'org.eclipse.jetty.npn.NextProtoNego$ServerProvider', + 'org.eclipse.jetty.npn.NextProtoNego', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty) + 'org.jboss.marshalling.ByteInput', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty) + 'org.jboss.marshalling.ByteOutput', + + // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty) + 'org.jboss.marshalling.Marshaller', + + // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty) + 'org.jboss.marshalling.MarshallerFactory', + 'org.jboss.marshalling.MarshallingConfiguration', + 'org.jboss.marshalling.Unmarshaller', + + // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + + 'com.google.protobuf.ExtensionRegistryLite', + 'com.google.protobuf.MessageLiteOrBuilder', + 'com.google.protobuf.nano.CodedOutputByteBufferNano', + 'com.google.protobuf.nano.MessageNano', + 'com.jcraft.jzlib.Deflater', + 'com.jcraft.jzlib.Inflater', + 'com.jcraft.jzlib.JZlib$WrapperType', + 'com.jcraft.jzlib.JZlib', + 'com.ning.compress.BufferRecycler', + 'com.ning.compress.lzf.ChunkDecoder', + 'com.ning.compress.lzf.ChunkEncoder', + 'com.ning.compress.lzf.LZFEncoder', + 'com.ning.compress.lzf.util.ChunkDecoderFactory', + 'com.ning.compress.lzf.util.ChunkEncoderFactory', + 'javassist/ClassClassPath', + 'javassist/ClassPath', + 'javassist/ClassPool', + 'javassist.CtClass', + 'javassist.CtMethod', + 'lzma.sdk.lzma.Encoder', + 'net.jpountz.lz4.LZ4Compressor', + 'net.jpountz.lz4.LZ4Factory', + 'net.jpountz.lz4.LZ4FastDecompressor', + 'net.jpountz.xxhash.StreamingXXHash32', + 'net.jpountz.xxhash.XXHashFactory', + 'org.apache.logging.log4j.LogManager', + 'org.apache.logging.log4j.Logger', + 'org.apache.tomcat.jni.CertificateRequestedCallback', + 'org.apache.tomcat.jni.CertificateVerifier', + 'org.apache.tomcat.jni.SessionTicketKey', + 'org.eclipse.jetty.alpn.ALPN$ClientProvider', + 'org.eclipse.jetty.alpn.ALPN$ServerProvider', + 'org.eclipse.jetty.alpn.ALPN', + + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator', + 'io.netty.util.internal.PlatformDependent0', + 'io.netty.util.internal.UnsafeAtomicIntegerFieldUpdater', + 'io.netty.util.internal.UnsafeAtomicLongFieldUpdater', + 'io.netty.util.internal.UnsafeAtomicReferenceFieldUpdater', + 'io.netty.util.internal.chmv8.ConcurrentHashMapV8', + 'io.netty.util.internal.chmv8.ConcurrentHashMapV8$1', + 'io.netty.util.internal.chmv8.ConcurrentHashMapV8$TreeBin', + 'io.netty.util.internal.chmv8.CountedCompleter', + 'io.netty.util.internal.chmv8.CountedCompleter$1', + 'io.netty.util.internal.chmv8.ForkJoinPool', + 'io.netty.util.internal.chmv8.ForkJoinPool$2', + 'io.netty.util.internal.chmv8.ForkJoinPool$WorkQueue', + 'io.netty.util.internal.chmv8.ForkJoinTask', + 'io.netty.util.internal.chmv8.ForkJoinTask$1', + 'io.netty.util.internal.chmv8.Striped64', + 'io.netty.util.internal.chmv8.Striped64$1', + 'io.netty.util.internal.chmv8.Striped64$Cell', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.ConcurrentSequencedCircularArrayQueue', + 'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueHeadLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueTailField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscChunkedArrayQueue', + 'io.netty.util.internal.shaded.org.jctools.util.JvmInfo', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', +] diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..e45d72cb77e --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +bae159f72e78d211733db0f13b0d8436a65af5ac \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-LICENSE.txt b/modules/transport-netty4/licenses/netty-buffer-LICENSE.txt similarity index 100% rename from client/sniffer/licenses/commons-codec-LICENSE.txt rename to modules/transport-netty4/licenses/netty-buffer-LICENSE.txt diff --git a/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt b/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..a1872082d0f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +a8fb268e5756768ed9ddc46806cf724481c14298 \ No newline at end of file diff --git a/client/sniffer/licenses/commons-logging-LICENSE.txt b/modules/transport-netty4/licenses/netty-codec-LICENSE.txt similarity index 100% rename from client/sniffer/licenses/commons-logging-LICENSE.txt rename to modules/transport-netty4/licenses/netty-codec-LICENSE.txt diff --git a/modules/transport-netty4/licenses/netty-codec-NOTICE.txt b/modules/transport-netty4/licenses/netty-codec-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..7c96920d161 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +8219258943eaa26115a8f39b16163b4d0e53770e \ No newline at end of file diff --git a/modules/reindex/licenses/commons-codec-LICENSE.txt b/modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt similarity index 100% rename from modules/reindex/licenses/commons-codec-LICENSE.txt rename to modules/transport-netty4/licenses/netty-codec-http-LICENSE.txt diff --git a/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt b/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-common-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..0e775de8650 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +8f2a9aafddf7a5db56e6c7dbbc94834e5bdeb186 \ No newline at end of file diff --git a/modules/reindex/licenses/commons-logging-LICENSE.txt b/modules/transport-netty4/licenses/netty-common-LICENSE.txt similarity index 100% rename from modules/reindex/licenses/commons-logging-LICENSE.txt rename to modules/transport-netty4/licenses/netty-common-LICENSE.txt index 57bc88a15a0..d6456956733 100644 --- a/modules/reindex/licenses/commons-logging-LICENSE.txt +++ b/modules/transport-netty4/licenses/netty-common-LICENSE.txt @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -199,4 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/modules/transport-netty4/licenses/netty-common-NOTICE.txt b/modules/transport-netty4/licenses/netty-common-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..e91cb888a44 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +99a2d3a7285195c7f42d25f236212b984d0dfcb7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-LICENSE.txt b/modules/transport-netty4/licenses/netty-handler-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-handler-NOTICE.txt b/modules/transport-netty4/licenses/netty-handler-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..951fcbf0f80 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +1f832049b94980a9f8d01ff87a83170c5a03f7eb \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt b/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt b/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 new file mode 100644 index 00000000000..4b7743836ca --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.4.Final-elastic-SNAPSHOT.jar.sha1 @@ -0,0 +1 @@ +16a2ffc9a952a8c1deccca462e7f9965608db2b2 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-LICENSE.txt b/modules/transport-netty4/licenses/netty-transport-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/transport-netty4/licenses/netty-transport-NOTICE.txt b/modules/transport-netty4/licenses/netty-transport-NOTICE.txt new file mode 100644 index 00000000000..5bbf91a14de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java new file mode 100644 index 00000000000..62443dc541e --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java @@ -0,0 +1,268 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.cookie.ServerCookieDecoder; +import io.netty.handler.codec.http.cookie.ServerCookieEncoder; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.netty4.cors.Netty4CorsHandler; +import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest; +import org.elasticsearch.rest.AbstractRestChannel; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +final class Netty4HttpChannel extends AbstractRestChannel { + + private final Netty4HttpServerTransport transport; + private final Channel channel; + private final FullHttpRequest nettyRequest; + private final HttpPipelinedRequest pipelinedRequest; + private final ThreadContext threadContext; + + /** + * @param transport The corresponding NettyHttpServerTransport where this channel belongs to. + * @param request The request that is handled by this channel. + * @param pipelinedRequest If HTTP pipelining is enabled provide the corresponding pipelined request. May be null if + * HTTP pipelining is disabled. + * @param detailedErrorsEnabled true iff error messages should include stack traces. + * @param threadContext the thread context for the channel + */ + Netty4HttpChannel( + final Netty4HttpServerTransport transport, + final Netty4HttpRequest request, + final HttpPipelinedRequest pipelinedRequest, + final boolean detailedErrorsEnabled, + final ThreadContext threadContext) { + super(request, detailedErrorsEnabled); + this.transport = transport; + this.channel = request.getChannel(); + this.nettyRequest = request.request(); + this.pipelinedRequest = pipelinedRequest; + this.threadContext = threadContext; + } + + @Override + public BytesStreamOutput newBytesOutput() { + return new ReleasableBytesStreamOutput(transport.bigArrays); + } + + + @Override + public void sendResponse(RestResponse response) { + // if the response object was created upstream, then use it; + // otherwise, create a new one + ByteBuf buffer = Netty4Utils.toByteBuf(response.content()); + FullHttpResponse resp = newResponse(buffer); + resp.setStatus(getStatus(response.status())); + + Netty4CorsHandler.setCorsResponseHeaders(nettyRequest, resp, transport.getCorsConfig()); + + String opaque = nettyRequest.headers().get("X-Opaque-Id"); + if (opaque != null) { + setHeaderField(resp, "X-Opaque-Id", opaque); + } + + // Add all custom headers + addCustomHeaders(resp, response.getHeaders()); + addCustomHeaders(resp, threadContext.getResponseHeaders()); + + BytesReference content = response.content(); + boolean release = content instanceof Releasable; + try { + // If our response doesn't specify a content-type header, set one + setHeaderField(resp, HttpHeaderNames.CONTENT_TYPE.toString(), response.contentType(), false); + // If our response has no content-length, calculate and set one + setHeaderField(resp, HttpHeaderNames.CONTENT_LENGTH.toString(), String.valueOf(buffer.readableBytes()), false); + + addCookies(resp); + + final ChannelPromise promise = channel.newPromise(); + + if (release) { + promise.addListener(f -> ((Releasable)content).close()); + release = false; + } + + if (isCloseConnection()) { + promise.addListener(ChannelFutureListener.CLOSE); + } + + if (pipelinedRequest != null) { + channel.writeAndFlush(pipelinedRequest.createHttpResponse(resp, promise)); + } else { + channel.writeAndFlush(resp, promise); + } + + } finally { + if (release) { + ((Releasable) content).close(); + } + if (pipelinedRequest != null) { + pipelinedRequest.release(); + } + } + } + + private void setHeaderField(HttpResponse resp, String headerField, String value) { + setHeaderField(resp, headerField, value, true); + } + + private void setHeaderField(HttpResponse resp, String headerField, String value, boolean override) { + if (override || !resp.headers().contains(headerField)) { + resp.headers().add(headerField, value); + } + } + + private void addCookies(HttpResponse resp) { + if (transport.resetCookies) { + String cookieString = nettyRequest.headers().get(HttpHeaders.Names.COOKIE); + if (cookieString != null) { + Set cookies = ServerCookieDecoder.STRICT.decode(cookieString); + if (!cookies.isEmpty()) { + // Reset the cookies if necessary. + resp.headers().set(HttpHeaderNames.SET_COOKIE, ServerCookieEncoder.STRICT.encode(cookies)); + } + } + } + } + + private void addCustomHeaders(HttpResponse response, Map> customHeaders) { + if (customHeaders != null) { + for (Map.Entry> headerEntry : customHeaders.entrySet()) { + for (String headerValue : headerEntry.getValue()) { + setHeaderField(response, headerEntry.getKey(), headerValue); + } + } + } + } + + // Determine if the request protocol version is HTTP 1.0 + private boolean isHttp10() { + return nettyRequest.protocolVersion().equals(HttpVersion.HTTP_1_0); + } + + // Determine if the request connection should be closed on completion. + private boolean isCloseConnection() { + final boolean http10 = isHttp10(); + return HttpHeaderValues.CLOSE.equals(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) || + (http10 && HttpHeaderValues.KEEP_ALIVE.equals(nettyRequest.headers().get(HttpHeaderNames.CONNECTION)) == false); + } + + // Create a new {@link HttpResponse} to transmit the response for the netty request. + private FullHttpResponse newResponse(ByteBuf buffer) { + final boolean http10 = isHttp10(); + final boolean close = isCloseConnection(); + // Build the response object. + final HttpResponseStatus status = HttpResponseStatus.OK; // default to initialize + final FullHttpResponse response; + if (http10) { + response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_0, status, buffer); + if (!close) { + response.headers().add(HttpHeaderNames.CONNECTION, "Keep-Alive"); + } + } else { + response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, buffer); + } + return response; + } + + private static final HttpResponseStatus TOO_MANY_REQUESTS = new HttpResponseStatus(429, "Too Many Requests"); + + private static Map MAP; + + static { + EnumMap map = new EnumMap<>(RestStatus.class); + map.put(RestStatus.CONTINUE, HttpResponseStatus.CONTINUE); + map.put(RestStatus.SWITCHING_PROTOCOLS, HttpResponseStatus.SWITCHING_PROTOCOLS); + map.put(RestStatus.OK, HttpResponseStatus.OK); + map.put(RestStatus.CREATED, HttpResponseStatus.CREATED); + map.put(RestStatus.ACCEPTED, HttpResponseStatus.ACCEPTED); + map.put(RestStatus.NON_AUTHORITATIVE_INFORMATION, HttpResponseStatus.NON_AUTHORITATIVE_INFORMATION); + map.put(RestStatus.NO_CONTENT, HttpResponseStatus.NO_CONTENT); + map.put(RestStatus.RESET_CONTENT, HttpResponseStatus.RESET_CONTENT); + map.put(RestStatus.PARTIAL_CONTENT, HttpResponseStatus.PARTIAL_CONTENT); + map.put(RestStatus.MULTI_STATUS, HttpResponseStatus.INTERNAL_SERVER_ERROR); // no status for this?? + map.put(RestStatus.MULTIPLE_CHOICES, HttpResponseStatus.MULTIPLE_CHOICES); + map.put(RestStatus.MOVED_PERMANENTLY, HttpResponseStatus.MOVED_PERMANENTLY); + map.put(RestStatus.FOUND, HttpResponseStatus.FOUND); + map.put(RestStatus.SEE_OTHER, HttpResponseStatus.SEE_OTHER); + map.put(RestStatus.NOT_MODIFIED, HttpResponseStatus.NOT_MODIFIED); + map.put(RestStatus.USE_PROXY, HttpResponseStatus.USE_PROXY); + map.put(RestStatus.TEMPORARY_REDIRECT, HttpResponseStatus.TEMPORARY_REDIRECT); + map.put(RestStatus.BAD_REQUEST, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.UNAUTHORIZED, HttpResponseStatus.UNAUTHORIZED); + map.put(RestStatus.PAYMENT_REQUIRED, HttpResponseStatus.PAYMENT_REQUIRED); + map.put(RestStatus.FORBIDDEN, HttpResponseStatus.FORBIDDEN); + map.put(RestStatus.NOT_FOUND, HttpResponseStatus.NOT_FOUND); + map.put(RestStatus.METHOD_NOT_ALLOWED, HttpResponseStatus.METHOD_NOT_ALLOWED); + map.put(RestStatus.NOT_ACCEPTABLE, HttpResponseStatus.NOT_ACCEPTABLE); + map.put(RestStatus.PROXY_AUTHENTICATION, HttpResponseStatus.PROXY_AUTHENTICATION_REQUIRED); + map.put(RestStatus.REQUEST_TIMEOUT, HttpResponseStatus.REQUEST_TIMEOUT); + map.put(RestStatus.CONFLICT, HttpResponseStatus.CONFLICT); + map.put(RestStatus.GONE, HttpResponseStatus.GONE); + map.put(RestStatus.LENGTH_REQUIRED, HttpResponseStatus.LENGTH_REQUIRED); + map.put(RestStatus.PRECONDITION_FAILED, HttpResponseStatus.PRECONDITION_FAILED); + map.put(RestStatus.REQUEST_ENTITY_TOO_LARGE, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE); + map.put(RestStatus.REQUEST_URI_TOO_LONG, HttpResponseStatus.REQUEST_URI_TOO_LONG); + map.put(RestStatus.UNSUPPORTED_MEDIA_TYPE, HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE); + map.put(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE); + map.put(RestStatus.EXPECTATION_FAILED, HttpResponseStatus.EXPECTATION_FAILED); + map.put(RestStatus.UNPROCESSABLE_ENTITY, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.LOCKED, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.FAILED_DEPENDENCY, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.TOO_MANY_REQUESTS, TOO_MANY_REQUESTS); + map.put(RestStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.INTERNAL_SERVER_ERROR); + map.put(RestStatus.NOT_IMPLEMENTED, HttpResponseStatus.NOT_IMPLEMENTED); + map.put(RestStatus.BAD_GATEWAY, HttpResponseStatus.BAD_GATEWAY); + map.put(RestStatus.SERVICE_UNAVAILABLE, HttpResponseStatus.SERVICE_UNAVAILABLE); + map.put(RestStatus.GATEWAY_TIMEOUT, HttpResponseStatus.GATEWAY_TIMEOUT); + map.put(RestStatus.HTTP_VERSION_NOT_SUPPORTED, HttpResponseStatus.HTTP_VERSION_NOT_SUPPORTED); + MAP = Collections.unmodifiableMap(map); + } + + private static HttpResponseStatus getStatus(RestStatus status) { + return MAP.getOrDefault(status, HttpResponseStatus.INTERNAL_SERVER_ERROR); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java new file mode 100644 index 00000000000..949faa848e7 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpRequest; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.transport.netty4.Netty4Utils; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.support.RestUtils; +import io.netty.channel.Channel; +import io.netty.handler.codec.http.HttpMethod; + +import java.net.SocketAddress; +import java.util.HashMap; +import java.util.Map; + +class Netty4HttpRequest extends RestRequest { + + private final FullHttpRequest request; + private final Channel channel; + private final BytesReference content; + + Netty4HttpRequest(FullHttpRequest request, Channel channel) { + super(request.uri()); + this.request = request; + this.channel = channel; + if (request.content().isReadable()) { + this.content = Netty4Utils.toBytesReference(request.content()); + } else { + this.content = BytesArray.EMPTY; + } + } + + public FullHttpRequest request() { + return this.request; + } + + @Override + public Method method() { + HttpMethod httpMethod = request.method(); + if (httpMethod == HttpMethod.GET) + return Method.GET; + + if (httpMethod == HttpMethod.POST) + return Method.POST; + + if (httpMethod == HttpMethod.PUT) + return Method.PUT; + + if (httpMethod == HttpMethod.DELETE) + return Method.DELETE; + + if (httpMethod == HttpMethod.HEAD) { + return Method.HEAD; + } + + if (httpMethod == HttpMethod.OPTIONS) { + return Method.OPTIONS; + } + + return Method.GET; + } + + @Override + public String uri() { + return request.uri(); + } + + @Override + public boolean hasContent() { + return content.length() > 0; + } + + @Override + public BytesReference content() { + return content; + } + + /** + * Returns the remote address where this rest request channel is "connected to". The + * returned {@link SocketAddress} is supposed to be down-cast into more + * concrete type such as {@link java.net.InetSocketAddress} to retrieve + * the detailed information. + */ + @Override + public SocketAddress getRemoteAddress() { + return channel.remoteAddress(); + } + + /** + * Returns the local address where this request channel is bound to. The returned + * {@link SocketAddress} is supposed to be down-cast into more concrete + * type such as {@link java.net.InetSocketAddress} to retrieve the detailed + * information. + */ + @Override + public SocketAddress getLocalAddress() { + return channel.localAddress(); + } + + public Channel getChannel() { + return channel; + } + + @Override + public String header(String name) { + return request.headers().get(name); + } + + @Override + public Iterable> headers() { + return request.headers().entries(); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java new file mode 100644 index 00000000000..a4402d4816f --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest; + +@ChannelHandler.Sharable +class Netty4HttpRequestHandler extends SimpleChannelInboundHandler { + + private final Netty4HttpServerTransport serverTransport; + private final boolean httpPipeliningEnabled; + private final boolean detailedErrorsEnabled; + private final ThreadContext threadContext; + + Netty4HttpRequestHandler(Netty4HttpServerTransport serverTransport, boolean detailedErrorsEnabled, ThreadContext threadContext) { + this.serverTransport = serverTransport; + this.httpPipeliningEnabled = serverTransport.pipelining; + this.detailedErrorsEnabled = detailedErrorsEnabled; + this.threadContext = threadContext; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception { + final FullHttpRequest request; + final HttpPipelinedRequest pipelinedRequest; + if (this.httpPipeliningEnabled && msg instanceof HttpPipelinedRequest) { + pipelinedRequest = (HttpPipelinedRequest) msg; + request = (FullHttpRequest) pipelinedRequest.last(); + } else { + pipelinedRequest = null; + request = (FullHttpRequest) msg; + } + + final FullHttpRequest copy = + new DefaultFullHttpRequest( + request.protocolVersion(), + request.method(), + request.uri(), + Unpooled.copiedBuffer(request.content()), + request.headers(), + request.trailingHeaders()); + + final Netty4HttpRequest httpRequest = new Netty4HttpRequest(copy, ctx.channel()); + serverTransport.dispatchRequest( + httpRequest, + new Netty4HttpChannel(serverTransport, httpRequest, pipelinedRequest, detailedErrorsEnabled, threadContext)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + serverTransport.exceptionCaught(ctx, cause); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java new file mode 100644 index 00000000000..910a499a194 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -0,0 +1,571 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import com.carrotsearch.hppc.IntHashSet; +import com.carrotsearch.hppc.IntSet; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.AdaptiveRecvByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.FixedRecvByteBufAllocator; +import io.netty.channel.RecvByteBufAllocator; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.oio.OioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.oio.OioServerSocketChannel; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.handler.codec.http.HttpContentCompressor; +import io.netty.handler.codec.http.HttpContentDecompressor; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequestDecoder; +import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.timeout.ReadTimeoutException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.NetworkExceptionHelper; +import org.elasticsearch.common.transport.PortsRange; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.BindHttpException; +import org.elasticsearch.http.HttpInfo; +import org.elasticsearch.http.HttpServerAdapter; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.HttpStats; +import org.elasticsearch.http.netty4.cors.Netty4CorsConfig; +import org.elasticsearch.http.netty4.cors.Netty4CorsConfigBuilder; +import org.elasticsearch.http.netty4.cors.Netty4CorsHandler; +import org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.support.RestUtils; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.BindTransportException; +import org.elasticsearch.transport.netty4.Netty4OpenChannelsHandler; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import static org.elasticsearch.common.settings.Setting.boolSetting; +import static org.elasticsearch.common.settings.Setting.byteSizeSetting; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_BIND_HOST; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PORT; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_RESET_COOKIES; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS; +import static org.elasticsearch.http.netty4.cors.Netty4CorsHandler.ANY_ORIGIN; + +public class Netty4HttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { + + static { + Netty4Utils.setup(); + } + + public static Setting SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY = + Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), + Property.NodeScope, Property.Shared); + public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = + Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope, Property.Shared); + + public static final Setting SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count", + (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), + (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope, Property.Shared); + + public static final Setting SETTING_HTTP_TCP_NO_DELAY = + boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_KEEP_ALIVE = + boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_BLOCKING_SERVER = + boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_REUSE_ADDRESS = + boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope, Property.Shared); + + public static final Setting SETTING_HTTP_TCP_SEND_BUFFER_SIZE = + Setting.byteSizeSetting("http.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE, + Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE = + Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE, + Property.NodeScope, Property.Shared); + public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE = + Setting.byteSizeSetting("transport.netty.receive_predictor_size", + settings -> { + long defaultReceiverPredictor = 512 * 1024; + if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { + // we can guess a better default... + long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / SETTING_HTTP_WORKER_COUNT.get + (settings)); + defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); + } + return new ByteSizeValue(defaultReceiverPredictor).toString(); + }, Property.NodeScope); + public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN = + byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + public static final Setting SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX = + byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope); + + + protected final NetworkService networkService; + protected final BigArrays bigArrays; + + protected final ByteSizeValue maxContentLength; + protected final ByteSizeValue maxInitialLineLength; + protected final ByteSizeValue maxHeaderSize; + protected final ByteSizeValue maxChunkSize; + + protected final int workerCount; + + protected final boolean blockingServer; + + protected final boolean pipelining; + + protected final int pipeliningMaxEvents; + + protected final boolean compression; + + protected final int compressionLevel; + + protected final boolean resetCookies; + + protected final PortsRange port; + + protected final String bindHosts[]; + + protected final String publishHosts[]; + + protected final boolean detailedErrorsEnabled; + protected final ThreadPool threadPool; + + protected final boolean tcpNoDelay; + protected final boolean tcpKeepAlive; + protected final boolean reuseAddress; + + protected final ByteSizeValue tcpSendBufferSize; + protected final ByteSizeValue tcpReceiveBufferSize; + protected final RecvByteBufAllocator recvByteBufAllocator; + + protected final ByteSizeValue maxCumulationBufferCapacity; + protected final int maxCompositeBufferComponents; + + protected volatile ServerBootstrap serverBootstrap; + + protected volatile BoundTransportAddress boundAddress; + + protected final List serverChannels = new ArrayList<>(); + + // package private for testing + Netty4OpenChannelsHandler serverOpenChannels; + + protected volatile HttpServerAdapter httpServerAdapter; + + private final Netty4CorsConfig corsConfig; + + @Inject + public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool) { + super(settings); + this.networkService = networkService; + this.bigArrays = bigArrays; + this.threadPool = threadPool; + + ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings); + this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings); + this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings); + this.maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings); + this.resetCookies = SETTING_HTTP_RESET_COOKIES.get(settings); + this.maxCumulationBufferCapacity = SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings); + this.maxCompositeBufferComponents = SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings); + this.workerCount = SETTING_HTTP_WORKER_COUNT.get(settings); + this.blockingServer = SETTING_HTTP_TCP_BLOCKING_SERVER.get(settings); + this.port = SETTING_HTTP_PORT.get(settings); + this.bindHosts = SETTING_HTTP_BIND_HOST.get(settings).toArray(Strings.EMPTY_ARRAY); + this.publishHosts = SETTING_HTTP_PUBLISH_HOST.get(settings).toArray(Strings.EMPTY_ARRAY); + this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings); + this.tcpKeepAlive = SETTING_HTTP_TCP_KEEP_ALIVE.get(settings); + this.reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings); + this.tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings); + this.tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings); + this.detailedErrorsEnabled = SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings); + + // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one + ByteSizeValue receivePredictorMin = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN.get(settings); + ByteSizeValue receivePredictorMax = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX.get(settings); + if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) { + recvByteBufAllocator = new FixedRecvByteBufAllocator(Math.toIntExact(receivePredictorMax.bytes())); + } else { + recvByteBufAllocator = new AdaptiveRecvByteBufAllocator( + Math.toIntExact(receivePredictorMin.bytes()), + Math.toIntExact(receivePredictorMin.bytes()), + Math.toIntExact(receivePredictorMax.bytes())); + } + + this.compression = SETTING_HTTP_COMPRESSION.get(settings); + this.compressionLevel = SETTING_HTTP_COMPRESSION_LEVEL.get(settings); + this.pipelining = SETTING_PIPELINING.get(settings); + this.pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings); + this.corsConfig = buildCorsConfig(settings); + + // validate max content length + if (maxContentLength.bytes() > Integer.MAX_VALUE) { + logger.warn("maxContentLength[{}] set to high value, resetting it to [100mb]", maxContentLength); + maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB); + } + this.maxContentLength = maxContentLength; + + logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " + + "receive_predictor[{}->{}], pipelining[{}], pipelining_max_events[{}]", + maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, + receivePredictorMin, receivePredictorMax, pipelining, pipeliningMaxEvents); + } + + public Settings settings() { + return this.settings; + } + + @Override + public void httpServerAdapter(HttpServerAdapter httpServerAdapter) { + this.httpServerAdapter = httpServerAdapter; + } + + @Override + protected void doStart() { + this.serverOpenChannels = new Netty4OpenChannelsHandler(logger); + + serverBootstrap = new ServerBootstrap(); + if (blockingServer) { + serverBootstrap.group(new OioEventLoopGroup(workerCount, daemonThreadFactory(settings, "http_server_worker"))); + serverBootstrap.channel(OioServerSocketChannel.class); + } else { + serverBootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, "http_server_worker"))); + serverBootstrap.channel(NioServerSocketChannel.class); + } + + serverBootstrap.childHandler(configureServerChannelHandler()); + + serverBootstrap.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings)); + serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings)); + + final ByteSizeValue tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings); + if (tcpSendBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes())); + } + + final ByteSizeValue tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings); + if (tcpReceiveBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytes())); + } + + serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + + final boolean reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings); + serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress); + serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress); + + this.boundAddress = createBoundHttpAddress(); + } + + private BoundTransportAddress createBoundHttpAddress() { + // Bind and start to accept incoming connections. + InetAddress hostAddresses[]; + try { + hostAddresses = networkService.resolveBindHostAddresses(bindHosts); + } catch (IOException e) { + throw new BindHttpException("Failed to resolve host [" + Arrays.toString(bindHosts) + "]", e); + } + + List boundAddresses = new ArrayList<>(hostAddresses.length); + for (InetAddress address : hostAddresses) { + boundAddresses.add(bindAddress(address)); + } + + final InetAddress publishInetAddress; + try { + publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts); + } catch (Exception e) { + throw new BindTransportException("Failed to resolve publish address", e); + } + + final int publishPort = resolvePublishPort(settings, boundAddresses, publishInetAddress); + final InetSocketAddress publishAddress = new InetSocketAddress(publishInetAddress, publishPort); + return new BoundTransportAddress(boundAddresses.toArray(new TransportAddress[0]), new InetSocketTransportAddress(publishAddress)); + } + + // package private for tests + static int resolvePublishPort(Settings settings, List boundAddresses, InetAddress publishInetAddress) { + int publishPort = SETTING_HTTP_PUBLISH_PORT.get(settings); + + if (publishPort < 0) { + for (InetSocketTransportAddress boundAddress : boundAddresses) { + InetAddress boundInetAddress = boundAddress.address().getAddress(); + if (boundInetAddress.isAnyLocalAddress() || boundInetAddress.equals(publishInetAddress)) { + publishPort = boundAddress.getPort(); + break; + } + } + } + + // if no matching boundAddress found, check if there is a unique port for all bound addresses + if (publishPort < 0) { + final IntSet ports = new IntHashSet(); + for (InetSocketTransportAddress boundAddress : boundAddresses) { + ports.add(boundAddress.getPort()); + } + if (ports.size() == 1) { + publishPort = ports.iterator().next().value; + } + } + + if (publishPort < 0) { + throw new BindHttpException("Failed to auto-resolve http publish port, multiple bound addresses " + boundAddresses + + " with distinct ports and none of them matched the publish address (" + publishInetAddress + "). " + + "Please specify a unique port by setting " + SETTING_HTTP_PORT.getKey() + " or " + SETTING_HTTP_PUBLISH_PORT.getKey()); + } + return publishPort; + } + + private Netty4CorsConfig buildCorsConfig(Settings settings) { + if (SETTING_CORS_ENABLED.get(settings) == false) { + return Netty4CorsConfigBuilder.forOrigins().disable().build(); + } + String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings); + final Netty4CorsConfigBuilder builder; + if (Strings.isNullOrEmpty(origin)) { + builder = Netty4CorsConfigBuilder.forOrigins(); + } else if (origin.equals(ANY_ORIGIN)) { + builder = Netty4CorsConfigBuilder.forAnyOrigin(); + } else { + Pattern p = RestUtils.checkCorsSettingForRegex(origin); + if (p == null) { + builder = Netty4CorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin)); + } else { + builder = Netty4CorsConfigBuilder.forPattern(p); + } + } + if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) { + builder.allowCredentials(); + } + String[] strMethods = settings.getAsArray(SETTING_CORS_ALLOW_METHODS.getKey()); + HttpMethod[] methods = Arrays.asList(strMethods) + .stream() + .map(HttpMethod::valueOf) + .toArray(size -> new HttpMethod[size]); + return builder.allowedRequestMethods(methods) + .maxAge(SETTING_CORS_MAX_AGE.get(settings)) + .allowedRequestHeaders(settings.getAsArray(SETTING_CORS_ALLOW_HEADERS.getKey())) + .shortCircuit() + .build(); + } + + private InetSocketTransportAddress bindAddress(final InetAddress hostAddress) { + final AtomicReference lastException = new AtomicReference<>(); + final AtomicReference boundSocket = new AtomicReference<>(); + boolean success = port.iterate(new PortsRange.PortCallback() { + @Override + public boolean onPortNumber(int portNumber) { + try { + synchronized (serverChannels) { + ChannelFuture future = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber)).sync(); + serverChannels.add(future.channel()); + boundSocket.set((InetSocketAddress) future.channel().localAddress()); + } + } catch (Exception e) { + lastException.set(e); + return false; + } + return true; + } + }); + if (!success) { + throw new BindHttpException("Failed to bind to [" + port + "]", lastException.get()); + } + + if (logger.isDebugEnabled()) { + logger.debug("Bound http to address {{}}", NetworkAddress.format(boundSocket.get())); + } + return new InetSocketTransportAddress(boundSocket.get()); + } + + @Override + protected void doStop() { + synchronized (serverChannels) { + if (!serverChannels.isEmpty()) { + try { + Netty4Utils.closeChannels(serverChannels); + } catch (IOException e) { + logger.trace("exception while closing channels", e); + } + serverChannels.clear(); + } + } + + if (serverOpenChannels != null) { + serverOpenChannels.close(); + serverOpenChannels = null; + } + + if (serverBootstrap != null) { + serverBootstrap.config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS).awaitUninterruptibly(); + serverBootstrap = null; + } + } + + @Override + protected void doClose() { + } + + @Override + public BoundTransportAddress boundAddress() { + return this.boundAddress; + } + + @Override + public HttpInfo info() { + BoundTransportAddress boundTransportAddress = boundAddress(); + if (boundTransportAddress == null) { + return null; + } + return new HttpInfo(boundTransportAddress, maxContentLength.bytes()); + } + + @Override + public HttpStats stats() { + Netty4OpenChannelsHandler channels = serverOpenChannels; + return new HttpStats(channels == null ? 0 : channels.numberOfOpenChannels(), channels == null ? 0 : channels.totalChannels()); + } + + public Netty4CorsConfig getCorsConfig() { + return corsConfig; + } + + protected void dispatchRequest(RestRequest request, RestChannel channel) { + httpServerAdapter.dispatchRequest(request, channel, threadPool.getThreadContext()); + } + + protected void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + if (cause instanceof ReadTimeoutException) { + if (logger.isTraceEnabled()) { + logger.trace("Connection timeout [{}]", ctx.channel().remoteAddress()); + } + ctx.channel().close(); + } else { + if (!lifecycle.started()) { + // ignore + return; + } + if (!NetworkExceptionHelper.isCloseConnectionException(cause)) { + logger.warn("caught exception while handling client http traffic, closing connection {}", cause, ctx.channel()); + ctx.channel().close(); + } else { + logger.debug("caught exception while handling client http traffic, closing connection {}", cause, ctx.channel()); + ctx.channel().close(); + } + } + } + + public ChannelHandler configureServerChannelHandler() { + return new HttpChannelHandler(this, detailedErrorsEnabled, threadPool.getThreadContext()); + } + + static class HttpChannelHandler extends ChannelInitializer { + + private final Netty4HttpServerTransport transport; + private final Netty4HttpRequestHandler requestHandler; + + HttpChannelHandler( + final Netty4HttpServerTransport transport, + final boolean detailedErrorsEnabled, + final ThreadContext threadContext) { + this.transport = transport; + this.requestHandler = new Netty4HttpRequestHandler(transport, detailedErrorsEnabled, threadContext); + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast("openChannels", transport.serverOpenChannels); + final HttpRequestDecoder decoder = new HttpRequestDecoder( + Math.toIntExact(transport.maxInitialLineLength.bytes()), + Math.toIntExact(transport.maxHeaderSize.bytes()), + Math.toIntExact(transport.maxChunkSize.bytes())); + decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR); + ch.pipeline().addLast("decoder", decoder); + ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor()); + final HttpObjectAggregator aggregator = new HttpObjectAggregator(Math.toIntExact(transport.maxContentLength.bytes())); + if (transport.maxCompositeBufferComponents != -1) { + aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); + } + ch.pipeline().addLast("aggregator", aggregator); + ch.pipeline().addLast("encoder", new HttpResponseEncoder()); + if (transport.compression) { + ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel)); + } + if (SETTING_CORS_ENABLED.get(transport.settings())) { + ch.pipeline().addLast("cors", new Netty4CorsHandler(transport.getCorsConfig())); + } + if (transport.pipelining) { + ch.pipeline().addLast("pipelining", new HttpPipeliningHandler(transport.pipeliningMaxEvents)); + } + ch.pipeline().addLast("handler", requestHandler); + } + + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java new file mode 100644 index 00000000000..9c81c07e663 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java @@ -0,0 +1,235 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.cors; + +import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.handler.codec.http.EmptyHttpHeaders; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.regex.Pattern; + +/** + * Configuration for Cross-Origin Resource Sharing (CORS). + * + * This class was lifted from the Netty project: + * https://github.com/netty/netty + */ +public final class Netty4CorsConfig { + + private final Optional> origins; + private final Optional pattern; + private final boolean anyOrigin; + private final boolean enabled; + private final boolean allowCredentials; + private final long maxAge; + private final Set allowedRequestMethods; + private final Set allowedRequestHeaders; + private final boolean allowNullOrigin; + private final Map> preflightHeaders; + private final boolean shortCircuit; + + Netty4CorsConfig(final Netty4CorsConfigBuilder builder) { + origins = builder.origins.map(s -> new LinkedHashSet<>(s)); + pattern = builder.pattern; + anyOrigin = builder.anyOrigin; + enabled = builder.enabled; + allowCredentials = builder.allowCredentials; + maxAge = builder.maxAge; + allowedRequestMethods = builder.requestMethods; + allowedRequestHeaders = builder.requestHeaders; + allowNullOrigin = builder.allowNullOrigin; + preflightHeaders = builder.preflightHeaders; + shortCircuit = builder.shortCircuit; + } + + /** + * Determines if support for CORS is enabled. + * + * @return {@code true} if support for CORS is enabled, false otherwise. + */ + public boolean isCorsSupportEnabled() { + return enabled; + } + + /** + * Determines whether a wildcard origin, '*', is supported. + * + * @return {@code boolean} true if any origin is allowed. + */ + public boolean isAnyOriginSupported() { + return anyOrigin; + } + + /** + * Returns the set of allowed origins. + * + * @return {@code Set} the allowed origins. + */ + public Optional> origins() { + return origins; + } + + /** + * Returns whether the input origin is allowed by this configuration. + * + * @return {@code true} if the origin is allowed, otherwise {@code false} + */ + public boolean isOriginAllowed(final String origin) { + if (origins.isPresent()) { + return origins.get().contains(origin); + } else if (pattern.isPresent()) { + return pattern.get().matcher(origin).matches(); + } + return false; + } + + /** + * Web browsers may set the 'Origin' request header to 'null' if a resource is loaded + * from the local file system. + * + * If isNullOriginAllowed is true then the server will response with the wildcard for the + * the CORS response header 'Access-Control-Allow-Origin'. + * + * @return {@code true} if a 'null' origin should be supported. + */ + public boolean isNullOriginAllowed() { + return allowNullOrigin; + } + + /** + * Determines if cookies are supported for CORS requests. + * + * By default cookies are not included in CORS requests but if isCredentialsAllowed returns + * true cookies will be added to CORS requests. Setting this value to true will set the + * CORS 'Access-Control-Allow-Credentials' response header to true. + * + * Please note that cookie support needs to be enabled on the client side as well. + * The client needs to opt-in to send cookies by calling: + * 
+     * xhr.withCredentials = true;
+     *
+ * The default value for 'withCredentials' is false in which case no cookies are sent. + * Setting this to true will included cookies in cross origin requests. + * + * @return {@code true} if cookies are supported. + */ + public boolean isCredentialsAllowed() { + return allowCredentials; + } + + /** + * Gets the maxAge setting. + * + * When making a preflight request the client has to perform two request with can be inefficient. + * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the + * caching of the preflight response for the specified time. During this time no preflight + * request will be made. + * + * @return {@code long} the time in seconds that a preflight request may be cached. + */ + public long maxAge() { + return maxAge; + } + + /** + * Returns the allowed set of Request Methods. The Http methods that should be returned in the + * CORS 'Access-Control-Request-Method' response header. + * + * @return {@code Set} of {@link HttpMethod}s that represent the allowed Request Methods. + */ + public Set allowedRequestMethods() { + return Collections.unmodifiableSet(allowedRequestMethods); + } + + /** + * Returns the allowed set of Request Headers. + * + * The header names returned from this method will be used to set the CORS + * 'Access-Control-Allow-Headers' response header. + * + * @return {@code Set} of strings that represent the allowed Request Headers. + */ + public Set allowedRequestHeaders() { + return Collections.unmodifiableSet(allowedRequestHeaders); + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * @return {@link HttpHeaders} the HTTP response headers to be added. + */ + public HttpHeaders preflightResponseHeaders() { + if (preflightHeaders.isEmpty()) { + return EmptyHttpHeaders.INSTANCE; + } + final HttpHeaders preflightHeaders = new DefaultHttpHeaders(); + for (Map.Entry> entry : this.preflightHeaders.entrySet()) { + final Object value = getValue(entry.getValue()); + if (value instanceof Iterable) { + preflightHeaders.add(entry.getKey().toString(), (Iterable) value); + } else { + preflightHeaders.add(entry.getKey().toString(), value); + } + } + return preflightHeaders; + } + + /** + * Determines whether a CORS request should be rejected if it's invalid before being + * further processing. + * + * CORS headers are set after a request is processed. This may not always be desired + * and this setting will check that the Origin is valid and if it is not valid no + * further processing will take place, and a error will be returned to the calling client. + * + * @return {@code true} if a CORS request should short-circuit upon receiving an invalid Origin header. + */ + public boolean isShortCircuit() { + return shortCircuit; + } + + private static T getValue(final Callable callable) { + try { + return callable.call(); + } catch (final Exception e) { + throw new IllegalStateException("Could not generate value for callable [" + callable + ']', e); + } + } + + @Override + public String toString() { + return "CorsConfig[enabled=" + enabled + + ", origins=" + origins + + ", anyOrigin=" + anyOrigin + + ", isCredentialsAllowed=" + allowCredentials + + ", maxAge=" + maxAge + + ", allowedRequestMethods=" + allowedRequestMethods + + ", allowedRequestHeaders=" + allowedRequestHeaders + + ", preflightHeaders=" + preflightHeaders + ']'; + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java new file mode 100644 index 00000000000..4989cd35f7b --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java @@ -0,0 +1,357 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.cors; + +import io.netty.handler.codec.http.HttpMethod; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.regex.Pattern; + +/** + * Builder used to configure and build a {@link Netty4CorsConfig} instance. + * + * This class was lifted from the Netty project: + * https://github.com/netty/netty + */ +public final class Netty4CorsConfigBuilder { + + /** + * Creates a Builder instance with it's origin set to '*'. + * + * @return Builder to support method chaining. + */ + public static Netty4CorsConfigBuilder forAnyOrigin() { + return new Netty4CorsConfigBuilder(); + } + + /** + * Creates a {@link Netty4CorsConfigBuilder} instance with the specified origin. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public static Netty4CorsConfigBuilder forOrigin(final String origin) { + if ("*".equals(origin)) { + return new Netty4CorsConfigBuilder(); + } + return new Netty4CorsConfigBuilder(origin); + } + + + /** + * Create a {@link Netty4CorsConfigBuilder} instance with the specified pattern origin. + * + * @param pattern the regular expression pattern to match incoming origins on. + * @return {@link Netty4CorsConfigBuilder} with the configured origin pattern. + */ + public static Netty4CorsConfigBuilder forPattern(final Pattern pattern) { + if (pattern == null) { + throw new IllegalArgumentException("CORS pattern cannot be null"); + } + return new Netty4CorsConfigBuilder(pattern); + } + + /** + * Creates a {@link Netty4CorsConfigBuilder} instance with the specified origins. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public static Netty4CorsConfigBuilder forOrigins(final String... origins) { + return new Netty4CorsConfigBuilder(origins); + } + + Optional> origins; + Optional pattern; + final boolean anyOrigin; + boolean allowNullOrigin; + boolean enabled = true; + boolean allowCredentials; + long maxAge; + final Set requestMethods = new HashSet<>(); + final Set requestHeaders = new HashSet<>(); + final Map> preflightHeaders = new HashMap<>(); + private boolean noPreflightHeaders; + boolean shortCircuit; + + /** + * Creates a new Builder instance with the origin passed in. + * + * @param origins the origin to be used for this builder. + */ + Netty4CorsConfigBuilder(final String... origins) { + this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins))); + pattern = Optional.empty(); + anyOrigin = false; + } + + /** + * Creates a new Builder instance allowing any origin, "*" which is the + * wildcard origin. + * + */ + Netty4CorsConfigBuilder() { + anyOrigin = true; + origins = Optional.empty(); + pattern = Optional.empty(); + } + + /** + * Creates a new Builder instance allowing any origin that matches the pattern. + * + * @param pattern the pattern to match against for incoming origins. + */ + Netty4CorsConfigBuilder(final Pattern pattern) { + this.pattern = Optional.of(pattern); + origins = Optional.empty(); + anyOrigin = false; + } + + /** + * Web browsers may set the 'Origin' request header to 'null' if a resource is loaded + * from the local file system. Calling this method will enable a successful CORS response + * with a wildcard for the CORS response header 'Access-Control-Allow-Origin'. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + Netty4CorsConfigBuilder allowNullOrigin() { + allowNullOrigin = true; + return this; + } + + /** + * Disables CORS support. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder disable() { + enabled = false; + return this; + } + + /** + * By default cookies are not included in CORS requests, but this method will enable cookies to + * be added to CORS requests. Calling this method will set the CORS 'Access-Control-Allow-Credentials' + * response header to true. + * + * Please note, that cookie support needs to be enabled on the client side as well. + * The client needs to opt-in to send cookies by calling: + * 
+     * xhr.withCredentials = true;
+     *
+ * The default value for 'withCredentials' is false in which case no cookies are sent. + * Setting this to true will included cookies in cross origin requests. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder allowCredentials() { + allowCredentials = true; + return this; + } + + /** + * When making a preflight request the client has to perform two request with can be inefficient. + * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the + * caching of the preflight response for the specified time. During this time no preflight + * request will be made. + * + * @param max the maximum time, in seconds, that the preflight response may be cached. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder maxAge(final long max) { + maxAge = max; + return this; + } + + /** + * Specifies the allowed set of HTTP Request Methods that should be returned in the + * CORS 'Access-Control-Request-Method' response header. + * + * @param methods the {@link HttpMethod}s that should be allowed. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder allowedRequestMethods(final HttpMethod... methods) { + requestMethods.addAll(Arrays.asList(methods)); + return this; + } + + /** + * Specifies the if headers that should be returned in the CORS 'Access-Control-Allow-Headers' + * response header. + * + * If a client specifies headers on the request, for example by calling: + * 
+     * xhr.setRequestHeader('My-Custom-Header', "SomeValue");
+     *
+ * the server will receive the above header name in the 'Access-Control-Request-Headers' of the + * preflight request. The server will then decide if it allows this header to be sent for the + * real request (remember that a preflight is not the real request but a request asking the server + * if it allow a request). + * + * @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder allowedRequestHeaders(final String... headers) { + requestHeaders.addAll(Arrays.asList(headers)); + return this; + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * An intermediary like a load balancer might require that a CORS preflight request + * have certain headers set. This enables such headers to be added. + * + * @param name the name of the HTTP header. + * @param values the values for the HTTP header. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Object... values) { + if (values.length == 1) { + preflightHeaders.put(name, new ConstantValueGenerator(values[0])); + } else { + preflightResponseHeader(name, Arrays.asList(values)); + } + return this; + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * An intermediary like a load balancer might require that a CORS preflight request + * have certain headers set. This enables such headers to be added. + * + * @param name the name of the HTTP header. + * @param value the values for the HTTP header. + * @param the type of values that the Iterable contains. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Iterable value) { + preflightHeaders.put(name, new ConstantValueGenerator(value)); + return this; + } + + /** + * Returns HTTP response headers that should be added to a CORS preflight response. + * + * An intermediary like a load balancer might require that a CORS preflight request + * have certain headers set. This enables such headers to be added. + * + * Some values must be dynamically created when the HTTP response is created, for + * example the 'Date' response header. This can be accomplished by using a Callable + * which will have its 'call' method invoked when the HTTP response is created. + * + * @param name the name of the HTTP header. + * @param valueGenerator a Callable which will be invoked at HTTP response creation. + * @param the type of the value that the Callable can return. + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder preflightResponseHeader(final CharSequence name, final Callable valueGenerator) { + preflightHeaders.put(name, valueGenerator); + return this; + } + + /** + * Specifies that no preflight response headers should be added to a preflight response. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder noPreflightResponseHeaders() { + noPreflightHeaders = true; + return this; + } + + /** + * Specifies that a CORS request should be rejected if it's invalid before being + * further processing. + * + * CORS headers are set after a request is processed. This may not always be desired + * and this setting will check that the Origin is valid and if it is not valid no + * further processing will take place, and a error will be returned to the calling client. + * + * @return {@link Netty4CorsConfigBuilder} to support method chaining. + */ + public Netty4CorsConfigBuilder shortCircuit() { + shortCircuit = true; + return this; + } + + /** + * Builds a {@link Netty4CorsConfig} with settings specified by previous method calls. + * + * @return {@link Netty4CorsConfig} the configured CorsConfig instance. + */ + public Netty4CorsConfig build() { + if (preflightHeaders.isEmpty() && !noPreflightHeaders) { + preflightHeaders.put("date", DateValueGenerator.INSTANCE); + preflightHeaders.put("content-length", new ConstantValueGenerator("0")); + } + return new Netty4CorsConfig(this); + } + + /** + * This class is used for preflight HTTP response values that do not need to be + * generated, but instead the value is "static" in that the same value will be returned + * for each call. + */ + private static final class ConstantValueGenerator implements Callable { + + private final Object value; + + /** + * Sole constructor. + * + * @param value the value that will be returned when the call method is invoked. + */ + private ConstantValueGenerator(final Object value) { + if (value == null) { + throw new IllegalArgumentException("value must not be null"); + } + this.value = value; + } + + @Override + public Object call() { + return value; + } + } + + /** + * This callable is used for the DATE preflight HTTP response HTTP header. + * It's value must be generated when the response is generated, hence will be + * different for every call. + */ + private static final class DateValueGenerator implements Callable { + + static final DateValueGenerator INSTANCE = new DateValueGenerator(); + + @Override + public Date call() throws Exception { + return new Date(); + } + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java new file mode 100644 index 00000000000..779eb4fe2e4 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java @@ -0,0 +1,235 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.cors; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import org.elasticsearch.common.Strings; + +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Handles Cross Origin Resource Sharing (CORS) requests. + *

+ * This handler can be configured using a {@link Netty4CorsConfig}, please + * refer to this class for details about the configuration options available. + * + * This code was borrowed from Netty 4 and refactored to work for Elasticsearch's Netty 3 setup. + */ +public class Netty4CorsHandler extends ChannelDuplexHandler { + + public static final String ANY_ORIGIN = "*"; + private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://"); + + private final Netty4CorsConfig config; + private HttpRequest request; + + /** + * Creates a new instance with the specified {@link Netty4CorsConfig}. + */ + public Netty4CorsHandler(final Netty4CorsConfig config) { + if (config == null) { + throw new NullPointerException(); + } + this.config = config; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (config.isCorsSupportEnabled() && msg instanceof HttpRequest) { + request = (HttpRequest) msg; + if (isPreflightRequest(request)) { + handlePreflight(ctx, request); + return; + } + if (config.isShortCircuit() && !validateOrigin()) { + forbidden(ctx, request); + return; + } + } + ctx.fireChannelRead(msg); + } + + public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, Netty4CorsConfig config) { + if (!config.isCorsSupportEnabled()) { + return; + } + String originHeader = request.headers().get(HttpHeaderNames.ORIGIN); + if (!Strings.isNullOrEmpty(originHeader)) { + final String originHeaderVal; + if (config.isAnyOriginSupported()) { + originHeaderVal = ANY_ORIGIN; + } else if (config.isOriginAllowed(originHeader) || isSameOrigin(originHeader, request.headers().get(HttpHeaderNames.HOST))) { + originHeaderVal = originHeader; + } else { + originHeaderVal = null; + } + if (originHeaderVal != null) { + resp.headers().add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, originHeaderVal); + } + } + if (config.isCredentialsAllowed()) { + resp.headers().add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true"); + } + } + + private void handlePreflight(final ChannelHandlerContext ctx, final HttpRequest request) { + final HttpResponse response = new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.OK, true, true); + if (setOrigin(response)) { + setAllowMethods(response); + setAllowHeaders(response); + setAllowCredentials(response); + setMaxAge(response); + setPreflightHeaders(response); + ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); + } else { + forbidden(ctx, request); + } + } + + private static void forbidden(final ChannelHandlerContext ctx, final HttpRequest request) { + ctx.writeAndFlush(new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.FORBIDDEN)) + .addListener(ChannelFutureListener.CLOSE); + } + + private static boolean isSameOrigin(final String origin, final String host) { + if (Strings.isNullOrEmpty(host) == false) { + // strip protocol from origin + final String originDomain = SCHEME_PATTERN.matcher(origin).replaceFirst(""); + if (host.equals(originDomain)) { + return true; + } + } + return false; + } + + /** + * This is a non CORS specification feature which enables the setting of preflight + * response headers that might be required by intermediaries. + * + * @param response the HttpResponse to which the preflight response headers should be added. + */ + private void setPreflightHeaders(final HttpResponse response) { + response.headers().add(config.preflightResponseHeaders()); + } + + private boolean setOrigin(final HttpResponse response) { + final String origin = request.headers().get(HttpHeaderNames.ORIGIN); + if (!Strings.isNullOrEmpty(origin)) { + if ("null".equals(origin) && config.isNullOriginAllowed()) { + setAnyOrigin(response); + return true; + } + + if (config.isAnyOriginSupported()) { + if (config.isCredentialsAllowed()) { + echoRequestOrigin(response); + setVaryHeader(response); + } else { + setAnyOrigin(response); + } + return true; + } + if (config.isOriginAllowed(origin)) { + setOrigin(response, origin); + setVaryHeader(response); + return true; + } + } + return false; + } + + private boolean validateOrigin() { + if (config.isAnyOriginSupported()) { + return true; + } + + final String origin = request.headers().get(HttpHeaderNames.ORIGIN); + if (Strings.isNullOrEmpty(origin)) { + // Not a CORS request so we cannot validate it. It may be a non CORS request. + return true; + } + + if ("null".equals(origin) && config.isNullOriginAllowed()) { + return true; + } + + // if the origin is the same as the host of the request, then allow + if (isSameOrigin(origin, request.headers().get(HttpHeaderNames.HOST))) { + return true; + } + + return config.isOriginAllowed(origin); + } + + private void echoRequestOrigin(final HttpResponse response) { + setOrigin(response, request.headers().get(HttpHeaderNames.ORIGIN)); + } + + private static void setVaryHeader(final HttpResponse response) { + response.headers().set(HttpHeaderNames.VARY, HttpHeaderNames.ORIGIN); + } + + private static void setAnyOrigin(final HttpResponse response) { + setOrigin(response, ANY_ORIGIN); + } + + private static void setOrigin(final HttpResponse response, final String origin) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, origin); + } + + private void setAllowCredentials(final HttpResponse response) { + if (config.isCredentialsAllowed() + && !response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN).equals(ANY_ORIGIN)) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true"); + } + } + + private static boolean isPreflightRequest(final HttpRequest request) { + final HttpHeaders headers = request.headers(); + return request.method().equals(HttpMethod.OPTIONS) && + headers.contains(HttpHeaderNames.ORIGIN) && + headers.contains(HttpHeaderNames.ACCESS_CONTROL_REQUEST_METHOD); + } + + private void setAllowMethods(final HttpResponse response) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS, config.allowedRequestMethods().stream() + .map(m -> m.name().trim()) + .collect(Collectors.toList())); + } + + private void setAllowHeaders(final HttpResponse response) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS, config.allowedRequestHeaders()); + } + + private void setMaxAge(final HttpResponse response) { + response.headers().set(HttpHeaderNames.ACCESS_CONTROL_MAX_AGE, config.maxAge()); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java new file mode 100644 index 00000000000..792397a3c3d --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedRequest.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.pipelining; + +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.ReferenceCounted; + +/** + * Permits downstream channel events to be ordered and signalled as to whether more are to come for + * a given sequence. + */ +public class HttpPipelinedRequest implements ReferenceCounted { + + private final LastHttpContent last; + private final int sequence; + + + HttpPipelinedRequest(final LastHttpContent last, final int sequence) { + this.last = last; + this.sequence = sequence; + } + + public LastHttpContent last() { + return last; + } + + public HttpPipelinedResponse createHttpResponse(final FullHttpResponse response, final ChannelPromise promise) { + return new HttpPipelinedResponse(response, promise, sequence); + } + + @Override + public int refCnt() { + return last.refCnt(); + } + + @Override + public ReferenceCounted retain() { + last.retain(); + return this; + } + + @Override + public ReferenceCounted retain(int increment) { + last.retain(increment); + return this; + } + + @Override + public ReferenceCounted touch() { + last.touch(); + return this; + } + + @Override + public ReferenceCounted touch(Object hint) { + last.touch(hint); + return this; + } + + @Override + public boolean release() { + return last.release(); + } + + @Override + public boolean release(int decrement) { + return last.release(decrement); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java new file mode 100644 index 00000000000..21659d5fbdf --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipelinedResponse.java @@ -0,0 +1,95 @@ +package org.elasticsearch.http.netty4.pipelining; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.util.ReferenceCounted; + +class HttpPipelinedResponse implements Comparable, ReferenceCounted { + + private final FullHttpResponse response; + private final ChannelPromise promise; + private final int sequence; + + HttpPipelinedResponse(FullHttpResponse response, ChannelPromise promise, int sequence) { + this.response = response; + this.promise = promise; + this.sequence = sequence; + } + + public FullHttpResponse response() { + return response; + } + + public ChannelPromise promise() { + return promise; + } + + public int sequence() { + return sequence; + } + + @Override + public int compareTo(HttpPipelinedResponse o) { + return Integer.compare(sequence, o.sequence); + } + + @Override + public int refCnt() { + return response.refCnt(); + } + + @Override + public ReferenceCounted retain() { + response.retain(); + return this; + } + + @Override + public ReferenceCounted retain(int increment) { + response.retain(increment); + return this; + } + + @Override + public ReferenceCounted touch() { + response.touch(); + return this; + } + + @Override + public ReferenceCounted touch(Object hint) { + response.touch(hint); + return this; + } + + @Override + public boolean release() { + return response.release(); + } + + @Override + public boolean release(int decrement) { + return response.release(decrement); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java new file mode 100644 index 00000000000..b96b7f5b322 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/pipelining/HttpPipeliningHandler.java @@ -0,0 +1,108 @@ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.pipelining; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.ReferenceCountUtil; +import org.elasticsearch.action.termvectors.TermVectorsFilter; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.util.Collections; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Implements HTTP pipelining ordering, ensuring that responses are completely served in the same order as their + * corresponding requests. NOTE: A side effect of using this handler is that upstream HttpRequest objects will + * cause the original message event to be effectively transformed into an OrderedUpstreamMessageEvent. Conversely + * OrderedDownstreamChannelEvent objects are expected to be received for the correlating response objects. + */ +public class HttpPipeliningHandler extends ChannelDuplexHandler { + + private static final int INITIAL_EVENTS_HELD = 3; + + private final int maxEventsHeld; + + private int readSequence; + private int writeSequence; + + private final Queue holdingQueue; + + /** + * @param maxEventsHeld the maximum number of channel events that will be retained prior to aborting the channel + * connection. This is required as events cannot queue up indefinitely; we would run out of + * memory if this was the case. + */ + public HttpPipeliningHandler(final int maxEventsHeld) { + this.maxEventsHeld = maxEventsHeld; + this.holdingQueue = new PriorityQueue<>(INITIAL_EVENTS_HELD); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof LastHttpContent) { + ctx.fireChannelRead(new HttpPipelinedRequest(((LastHttpContent) msg).retain(), readSequence++)); + } else { + ctx.fireChannelRead(msg); + } + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof HttpPipelinedResponse) { + boolean channelShouldClose = false; + + synchronized (holdingQueue) { + if (holdingQueue.size() < maxEventsHeld) { + holdingQueue.add((HttpPipelinedResponse) msg); + + while (!holdingQueue.isEmpty()) { + final HttpPipelinedResponse response = holdingQueue.peek(); + if (response.sequence() != writeSequence) { + break; + } + holdingQueue.remove(); + ctx.write(response.response(), response.promise()); + writeSequence++; + } + } else { + channelShouldClose = true; + } + } + + if (channelShouldClose) { + try { + Netty4Utils.closeChannels(Collections.singletonList(ctx.channel())); + } finally { + ((HttpPipelinedResponse) msg).release(); + promise.setSuccess(); + } + } + } else { + ctx.write(msg, promise); + } + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java new file mode 100644 index 00000000000..1ec2852fd1d --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.http.netty4.Netty4HttpServerTransport; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Transport; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Arrays; +import java.util.List; + +public class Netty4Plugin extends Plugin { + + public static final String NETTY_TRANSPORT_NAME = "netty4"; + public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4"; + + public Netty4Plugin(Settings settings) { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + Class.forName("io.netty.channel.nio.NioEventLoop"); + } catch (ClassNotFoundException e) { + throw new AssertionError(e); // we don't do anything with this + } + return null; + }); + /* + * Asserts that sun.nio.ch.bugLevel has been set to a non-null value. This assertion will fail if the corresponding code + * is not executed in a doPrivileged block. This can be disabled via `netty.assert.buglevel` setting which isn't registered + * by default but test can do so if they depend on the jar instead of the module. + */ + //TODO Once we have no jar level dependency we can get rid of this. + if (settings.getAsBoolean("netty.assert.buglevel", true)) { + assert System.getProperty("sun.nio.ch.bugLevel") != null : + "sun.nio.ch.bugLevel is null somebody pulls in SelectorUtil without doing stuff in a doPrivileged block?"; + } + } + + @Override + public List> getSettings() { + return Arrays.asList( + Netty4HttpServerTransport.SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY, + Netty4HttpServerTransport.SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, + Netty4HttpServerTransport.SETTING_HTTP_WORKER_COUNT, + Netty4HttpServerTransport.SETTING_HTTP_TCP_NO_DELAY, + Netty4HttpServerTransport.SETTING_HTTP_TCP_KEEP_ALIVE, + Netty4HttpServerTransport.SETTING_HTTP_TCP_BLOCKING_SERVER, + Netty4HttpServerTransport.SETTING_HTTP_TCP_REUSE_ADDRESS, + Netty4HttpServerTransport.SETTING_HTTP_TCP_SEND_BUFFER_SIZE, + Netty4HttpServerTransport.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE, + Netty4Transport.WORKER_COUNT, + Netty4Transport.NETTY_MAX_CUMULATION_BUFFER_CAPACITY, + Netty4Transport.NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, + Netty4Transport.NETTY_RECEIVE_PREDICTOR_SIZE, + Netty4Transport.NETTY_RECEIVE_PREDICTOR_MIN, + Netty4Transport.NETTY_RECEIVE_PREDICTOR_MAX, + Netty4Transport.NETTY_BOSS_COUNT + ); + } + + public void onModule(NetworkModule networkModule) { + if (networkModule.canRegisterHttpExtensions()) { + networkModule.registerHttpTransport(NETTY_HTTP_TRANSPORT_NAME, Netty4HttpServerTransport.class); + } + networkModule.registerTransport(NETTY_TRANSPORT_NAME, Netty4Transport.class); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java new file mode 100644 index 00000000000..3b4aba10281 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +final class ByteBufBytesReference extends BytesReference { + + private final ByteBuf buffer; + private final int length; + private final int offset; + + ByteBufBytesReference(ByteBuf buffer, int length) { + this.buffer = buffer; + this.length = length; + this.offset = buffer.readerIndex(); + assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes(); + } + + @Override + public byte get(int index) { + return buffer.getByte(offset + index); + } + + @Override + public int length() { + return length; + } + + @Override + public BytesReference slice(int from, int length) { + return new ByteBufBytesReference(buffer.slice(offset + from, length), length); + } + + @Override + public StreamInput streamInput() { + return new ByteBufStreamInput(buffer.duplicate(), length); + } + + @Override + public void writeTo(OutputStream os) throws IOException { + buffer.getBytes(offset, os, length); + } + + ByteBuf toByteBuf() { + return buffer.duplicate(); + } + + @Override + public String utf8ToString() { + return buffer.toString(offset, length, StandardCharsets.UTF_8); + } + + @Override + public BytesRef toBytesRef() { + if (buffer.hasArray()) { + return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length); + } + final byte[] copy = new byte[length]; + buffer.getBytes(offset, copy); + return new BytesRef(copy); + } + + @Override + public long ramBytesUsed() { + return buffer.capacity(); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java new file mode 100644 index 00000000000..2219ce31ff6 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * A Netty {@link io.netty.buffer.ByteBuf} based {@link org.elasticsearch.common.io.stream.StreamInput}. + */ +class ByteBufStreamInput extends StreamInput { + + private final ByteBuf buffer; + private final int startIndex; + private final int endIndex; + + public ByteBufStreamInput(ByteBuf buffer, int length) { + if (length > buffer.readableBytes()) { + throw new IndexOutOfBoundsException(); + } + this.buffer = buffer; + startIndex = buffer.readerIndex(); + endIndex = startIndex + length; + buffer.markReaderIndex(); + } + + @Override + public BytesReference readBytesReference(int length) throws IOException { + BytesReference ref = Netty4Utils.toBytesReference(buffer.slice(buffer.readerIndex(), length)); + buffer.skipBytes(length); + return ref; + } + + @Override + public BytesRef readBytesRef(int length) throws IOException { + if (!buffer.hasArray()) { + return super.readBytesRef(length); + } + BytesRef bytesRef = new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), length); + buffer.skipBytes(length); + return bytesRef; + } + + @Override + public int available() throws IOException { + return endIndex - buffer.readerIndex(); + } + + @Override + public void mark(int readlimit) { + buffer.markReaderIndex(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public int read() throws IOException { + if (available() == 0) { + return -1; + } + return buffer.readByte() & 0xff; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (len == 0) { + return 0; + } + int available = available(); + if (available == 0) { + return -1; + } + + len = Math.min(available, len); + buffer.readBytes(b, off, len); + return len; + } + + @Override + public void reset() throws IOException { + buffer.resetReaderIndex(); + } + + @Override + public long skip(long n) throws IOException { + if (n > Integer.MAX_VALUE) { + return skipBytes(Integer.MAX_VALUE); + } else { + return skipBytes((int) n); + } + } + + public int skipBytes(int n) throws IOException { + int nBytes = Math.min(available(), n); + buffer.skipBytes(nBytes); + return nBytes; + } + + + @Override + public byte readByte() throws IOException { + return buffer.readByte(); + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + int read = read(b, offset, len); + if (read < len) { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void close() throws IOException { + // nothing to do here + } +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java new file mode 100644 index 00000000000..61555294018 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.util.internal.logging.AbstractInternalLogger; +import org.elasticsearch.common.SuppressLoggerChecks; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; + +@SuppressLoggerChecks(reason = "safely delegates to logger") +class Netty4InternalESLogger extends AbstractInternalLogger { + + private final ESLogger logger; + + Netty4InternalESLogger(final String name) { + super(name); + this.logger = Loggers.getLogger(name); + } + + @Override + public boolean isTraceEnabled() { + return logger.isTraceEnabled(); + } + + @Override + public void trace(String msg) { + logger.trace(msg); + } + + @Override + public void trace(String format, Object arg) { + logger.trace(format, arg); + } + + @Override + public void trace(String format, Object argA, Object argB) { + logger.trace(format, argA, argB); + } + + @Override + public void trace(String format, Object... arguments) { + logger.trace(format, arguments); + } + + @Override + public void trace(String msg, Throwable t) { + logger.trace(msg, t); + } + + @Override + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } + + @Override + public void debug(String msg) { + logger.debug(msg); + } + + @Override + public void debug(String format, Object arg) { + logger.debug(format, arg); + } + + @Override + public void debug(String format, Object argA, Object argB) { + logger.debug(format, argA, argB); + } + + @Override + public void debug(String format, Object... arguments) { + logger.debug(format, arguments); + } + + @Override + public void debug(String msg, Throwable t) { + logger.debug(msg, t); + } + + @Override + public boolean isInfoEnabled() { + return logger.isInfoEnabled(); + } + + @Override + public void info(String msg) { + logger.info(msg); + } + + @Override + public void info(String format, Object arg) { + logger.info(format, arg); + } + + @Override + public void info(String format, Object argA, Object argB) { + logger.info(format, argA, argB); + } + + @Override + public void info(String format, Object... arguments) { + logger.info(format, arguments); + } + + @Override + public void info(String msg, Throwable t) { + logger.info(msg, t); + } + + @Override + public boolean isWarnEnabled() { + return logger.isWarnEnabled(); + } + + @Override + public void warn(String msg) { + logger.warn(msg); + } + + @Override + public void warn(String format, Object arg) { + logger.warn(format, arg); + } + + @Override + public void warn(String format, Object... arguments) { + logger.warn(format, arguments); + } + + @Override + public void warn(String format, Object argA, Object argB) { + logger.warn(format, argA, argB); + } + + @Override + public void warn(String msg, Throwable t) { + logger.warn(msg, t); + } + + @Override + public boolean isErrorEnabled() { + return logger.isErrorEnabled(); + } + + @Override + public void error(String msg) { + logger.equals(msg); + } + + @Override + public void error(String format, Object arg) { + logger.error(format, arg); + } + + @Override + public void error(String format, Object argA, Object argB) { + logger.error(format, argA, argB); + } + + @Override + public void error(String format, Object... arguments) { + logger.error(format, arguments); + } + + @Override + public void error(String msg, Throwable t) { + logger.error(msg, t); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java new file mode 100644 index 00000000000..cddbdac3e76 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.util.ReferenceCountUtil; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.transport.TcpHeader; +import org.elasticsearch.transport.TransportServiceAdapter; +import org.elasticsearch.transport.Transports; + +import java.net.InetSocketAddress; + +/** + * A handler (must be the last one!) that does size based frame decoding and forwards the actual message + * to the relevant action. + */ +final class Netty4MessageChannelHandler extends ChannelDuplexHandler { + + private final TransportServiceAdapter transportServiceAdapter; + private final Netty4Transport transport; + private final String profileName; + + Netty4MessageChannelHandler(Netty4Transport transport, String profileName) { + this.transportServiceAdapter = transport.transportServiceAdapter(); + this.transport = transport; + this.profileName = profileName; + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof ByteBuf && transportServiceAdapter != null) { + // record the number of bytes send on the channel + promise.addListener(f -> transportServiceAdapter.addBytesSent(((ByteBuf) msg).readableBytes())); + } + ctx.write(msg, promise); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + Transports.assertTransportThread(); + if (!(msg instanceof ByteBuf)) { + ctx.fireChannelRead(msg); + return; + } + final ByteBuf buffer = (ByteBuf) msg; + final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE); + final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize; + InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress(); + try { + // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh + // buffer, or in the cumulation buffer, which is cleaned each time so it could be bigger than the actual size + BytesReference reference = Netty4Utils.toBytesReference(buffer, remainingMessageSize); + transport.messageReceived(reference, ctx.channel(), profileName, remoteAddress, remainingMessageSize); + } finally { + // Set the expected position of the buffer, no matter what happened + buffer.readerIndex(expectedReaderIndex); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + transport.exceptionCaught(ctx, cause); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4OpenChannelsHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4OpenChannelsHandler.java new file mode 100644 index 00000000000..0562a0d4661 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4OpenChannelsHandler.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.metrics.CounterMetric; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@ChannelHandler.Sharable +public class Netty4OpenChannelsHandler extends ChannelInboundHandlerAdapter implements Releasable { + + final Set openChannels = Collections.newSetFromMap(new ConcurrentHashMap<>()); + final CounterMetric openChannelsMetric = new CounterMetric(); + final CounterMetric totalChannelsMetric = new CounterMetric(); + + final ESLogger logger; + + public Netty4OpenChannelsHandler(ESLogger logger) { + this.logger = logger; + } + + final ChannelFutureListener remover = new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + boolean removed = openChannels.remove(future.channel()); + if (removed) { + openChannelsMetric.dec(); + } + if (logger.isTraceEnabled()) { + logger.trace("channel closed: {}", future.channel()); + } + } + }; + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + if (logger.isTraceEnabled()) { + logger.trace("channel opened: {}", ctx.channel()); + } + final boolean added = openChannels.add(ctx.channel()); + if (added) { + openChannelsMetric.inc(); + totalChannelsMetric.inc(); + ctx.channel().closeFuture().addListener(remover); + } + + super.channelActive(ctx); + } + + public long numberOfOpenChannels() { + return openChannelsMetric.count(); + } + + public long totalChannels() { + return totalChannelsMetric.count(); + } + + @Override + public void close() { + try { + Netty4Utils.closeChannels(openChannels); + } catch (IOException e) { + logger.trace("exception while closing channels", e); + } + openChannels.clear(); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java new file mode 100644 index 00000000000..144c036f08a --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.handler.codec.TooLongFrameException; +import org.elasticsearch.transport.TcpHeader; +import org.elasticsearch.transport.TcpTransport; + +import java.util.List; + +final class Netty4SizeHeaderFrameDecoder extends ByteToMessageDecoder { + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + try { + boolean continueProcessing = TcpTransport.validateMessageHeader(Netty4Utils.toBytesReference(in)); + final ByteBuf message = in.skipBytes(TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE); + if (!continueProcessing) return; + out.add(message); + } catch (IllegalArgumentException ex) { + throw new TooLongFrameException(ex); + } catch (IllegalStateException ex) { + /* decode will be called until the ByteBuf is fully consumed; when it is fully + * consumed, transport#validateMessageHeader will throw an IllegalStateException which + * is okay, it means we have finished consuming the ByteBuf and we can get out + */ + } + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java new file mode 100644 index 00000000000..c1b2ef10211 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java @@ -0,0 +1,506 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.AdaptiveRecvByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.FixedRecvByteBufAllocator; +import io.netty.channel.RecvByteBufAllocator; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.oio.OioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.channel.socket.oio.OioServerSocketChannel; +import io.netty.channel.socket.oio.OioSocketChannel; +import io.netty.util.concurrent.Future; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.network.NetworkService.TcpSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportServiceAdapter; +import org.elasticsearch.transport.TransportSettings; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.common.settings.Setting.byteSizeSetting; +import static org.elasticsearch.common.settings.Setting.intSetting; +import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; + +/** + * There are 4 types of connections per node, low/med/high/ping. Low if for batch oriented APIs (like recovery or + * batch) with high payload that will cause regular request. (like search or single index) to take + * longer. Med is for the typical search / single doc index. And High for things like cluster state. Ping is reserved for + * sending out ping requests to other nodes. + */ +public class Netty4Transport extends TcpTransport { + + static { + Netty4Utils.setup(); + } + + public static final Setting WORKER_COUNT = + new Setting<>("transport.netty.worker_count", + (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2), + (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope, Property.Shared); + + public static final Setting NETTY_MAX_CUMULATION_BUFFER_CAPACITY = + Setting.byteSizeSetting( + "transport.netty.max_cumulation_buffer_capacity", + new ByteSizeValue(-1), + Property.NodeScope, + Property.Shared); + public static final Setting NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = + Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope, Property.Shared); + + // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one + public static final Setting NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting( + "transport.netty.receive_predictor_size", + settings -> { + long defaultReceiverPredictor = 512 * 1024; + if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) { + // we can guess a better default... + long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings)); + defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024)); + } + return new ByteSizeValue(defaultReceiverPredictor).toString(); + }, + Property.NodeScope, + Property.Shared); + public static final Setting NETTY_RECEIVE_PREDICTOR_MIN = + byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); + public static final Setting NETTY_RECEIVE_PREDICTOR_MAX = + byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope, Property.Shared); + public static final Setting NETTY_BOSS_COUNT = + intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope, Property.Shared); + + + protected final ByteSizeValue maxCumulationBufferCapacity; + protected final int maxCompositeBufferComponents; + protected final RecvByteBufAllocator recvByteBufAllocator; + protected final int workerCount; + protected final ByteSizeValue receivePredictorMin; + protected final ByteSizeValue receivePredictorMax; + // package private for testing + volatile Netty4OpenChannelsHandler serverOpenChannels; + protected volatile Bootstrap bootstrap; + protected final Map serverBootstraps = newConcurrentMap(); + + @Inject + public Netty4Transport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays, + NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) { + super("netty", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService); + this.workerCount = WORKER_COUNT.get(settings); + this.maxCumulationBufferCapacity = NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings); + this.maxCompositeBufferComponents = NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings); + + // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one + this.receivePredictorMin = NETTY_RECEIVE_PREDICTOR_MIN.get(settings); + this.receivePredictorMax = NETTY_RECEIVE_PREDICTOR_MAX.get(settings); + if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) { + recvByteBufAllocator = new FixedRecvByteBufAllocator((int) receivePredictorMax.bytes()); + } else { + recvByteBufAllocator = new AdaptiveRecvByteBufAllocator((int) receivePredictorMin.bytes(), + (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes()); + } + } + + TransportServiceAdapter transportServiceAdapter() { + return transportServiceAdapter; + } + + @Override + protected void doStart() { + boolean success = false; + try { + bootstrap = createBootstrap(); + if (NetworkService.NETWORK_SERVER.get(settings)) { + final Netty4OpenChannelsHandler openChannels = new Netty4OpenChannelsHandler(logger); + this.serverOpenChannels = openChannels; + // loop through all profiles and start them up, special handling for default one + for (Map.Entry entry : buildProfileSettings().entrySet()) { + // merge fallback settings with default settings with profile settings so we have complete settings with default values + final Settings settings = Settings.builder() + .put(createFallbackSettings()) + .put(entry.getValue()).build(); + createServerBootstrap(entry.getKey(), settings); + bindServer(entry.getKey(), settings); + } + } + super.doStart(); + success = true; + } finally { + if (success == false) { + doStop(); + } + } + } + + private Bootstrap createBootstrap() { + final Bootstrap bootstrap = new Bootstrap(); + if (TCP_BLOCKING_CLIENT.get(settings)) { + bootstrap.group(new OioEventLoopGroup(1, daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))); + bootstrap.channel(OioSocketChannel.class); + } else { + bootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX))); + bootstrap.channel(NioSocketChannel.class); + } + + bootstrap.handler(new ChannelInitializer() { + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder()); + // using a dot as a prefix means this cannot come from any settings parsed + ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, ".client")); + } + + }); + + bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(connectTimeout.millis())); + bootstrap.option(ChannelOption.TCP_NODELAY, TCP_NO_DELAY.get(settings)); + bootstrap.option(ChannelOption.SO_KEEPALIVE, TCP_KEEP_ALIVE.get(settings)); + + final ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.get(settings); + if (tcpSendBufferSize.bytes() > 0) { + bootstrap.option(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes())); + } + + final ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings); + if (tcpReceiveBufferSize.bytes() > 0) { + bootstrap.option(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytes())); + } + + bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + + final boolean reuseAddress = TCP_REUSE_ADDRESS.get(settings); + bootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress); + + bootstrap.validate(); + + return bootstrap; + } + + private Settings createFallbackSettings() { + Settings.Builder fallbackSettingsBuilder = Settings.builder(); + + List fallbackBindHost = TransportSettings.BIND_HOST.get(settings); + if (fallbackBindHost.isEmpty() == false) { + fallbackSettingsBuilder.putArray("bind_host", fallbackBindHost); + } + + List fallbackPublishHost = TransportSettings.PUBLISH_HOST.get(settings); + if (fallbackPublishHost.isEmpty() == false) { + fallbackSettingsBuilder.putArray("publish_host", fallbackPublishHost); + } + + boolean fallbackTcpNoDelay = settings.getAsBoolean("transport.netty.tcp_no_delay", TcpSettings.TCP_NO_DELAY.get(settings)); + fallbackSettingsBuilder.put("tcp_no_delay", fallbackTcpNoDelay); + + boolean fallbackTcpKeepAlive = settings.getAsBoolean("transport.netty.tcp_keep_alive", TcpSettings.TCP_KEEP_ALIVE.get(settings)); + fallbackSettingsBuilder.put("tcp_keep_alive", fallbackTcpKeepAlive); + + boolean fallbackReuseAddress = settings.getAsBoolean("transport.netty.reuse_address", TcpSettings.TCP_REUSE_ADDRESS.get(settings)); + fallbackSettingsBuilder.put("reuse_address", fallbackReuseAddress); + + ByteSizeValue fallbackTcpSendBufferSize = settings.getAsBytesSize("transport.netty.tcp_send_buffer_size", + TCP_SEND_BUFFER_SIZE.get(settings)); + if (fallbackTcpSendBufferSize.bytes() >= 0) { + fallbackSettingsBuilder.put("tcp_send_buffer_size", fallbackTcpSendBufferSize); + } + + ByteSizeValue fallbackTcpBufferSize = settings.getAsBytesSize("transport.netty.tcp_receive_buffer_size", + TCP_RECEIVE_BUFFER_SIZE.get(settings)); + if (fallbackTcpBufferSize.bytes() >= 0) { + fallbackSettingsBuilder.put("tcp_receive_buffer_size", fallbackTcpBufferSize); + } + + return fallbackSettingsBuilder.build(); + } + + private void createServerBootstrap(String name, Settings settings) { + if (logger.isDebugEnabled()) { + logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], " + + "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]", + name, workerCount, settings.get("port"), settings.get("bind_host"), settings.get("publish_host"), compress, + connectTimeout, connectionsPerNodeRecovery, connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState, + connectionsPerNodePing, receivePredictorMin, receivePredictorMax); + } + + final ThreadFactory workerFactory = daemonThreadFactory(this.settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX, name); + + final ServerBootstrap serverBootstrap = new ServerBootstrap(); + + if (TCP_BLOCKING_SERVER.get(settings)) { + serverBootstrap.group(new OioEventLoopGroup(workerCount, workerFactory)); + serverBootstrap.channel(OioServerSocketChannel.class); + } else { + serverBootstrap.group(new NioEventLoopGroup(workerCount, workerFactory)); + serverBootstrap.channel(NioServerSocketChannel.class); + } + + serverBootstrap.childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast("open_channels", Netty4Transport.this.serverOpenChannels); + ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder()); + ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, name)); + } + }); + + serverBootstrap.childOption(ChannelOption.TCP_NODELAY, TCP_NO_DELAY.get(settings)); + serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, TCP_KEEP_ALIVE.get(settings)); + + final ByteSizeValue tcpSendBufferSize = TCP_SEND_BUFFER_SIZE.getDefault(settings); + if (tcpSendBufferSize != null && tcpSendBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.bytes())); + } + + final ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.getDefault(settings); + if (tcpReceiveBufferSize != null && tcpReceiveBufferSize.bytes() > 0) { + serverBootstrap.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.bytesAsInt())); + } + + serverBootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + serverBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, recvByteBufAllocator); + + final boolean reuseAddress = TCP_REUSE_ADDRESS.get(settings); + serverBootstrap.option(ChannelOption.SO_REUSEADDR, reuseAddress); + serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress); + + serverBootstrap.validate(); + + serverBootstraps.put(name, serverBootstrap); + } + + protected final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final Throwable unwrapped = ExceptionsHelper.unwrap(cause, ElasticsearchException.class); + final Throwable t = unwrapped != null ? unwrapped : cause; + onException(ctx.channel(), t instanceof Exception ? (Exception) t : new ElasticsearchException(t)); + } + + @Override + public long serverOpen() { + Netty4OpenChannelsHandler channels = serverOpenChannels; + return channels == null ? 0 : channels.numberOfOpenChannels(); + } + + protected NodeChannels connectToChannelsLight(DiscoveryNode node) { + InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address(); + ChannelFuture connect = bootstrap.connect(address); + connect.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5)); + if (!connect.isSuccess()) { + throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connect.cause()); + } + Channel[] channels = new Channel[1]; + channels[0] = connect.channel(); + channels[0].closeFuture().addListener(new ChannelCloseListener(node)); + return new NodeChannels(channels, channels, channels, channels, channels); + } + + protected NodeChannels connectToChannels(DiscoveryNode node) { + final NodeChannels nodeChannels = + new NodeChannels( + new Channel[connectionsPerNodeRecovery], + new Channel[connectionsPerNodeBulk], + new Channel[connectionsPerNodeReg], + new Channel[connectionsPerNodeState], + new Channel[connectionsPerNodePing]); + boolean success = false; + try { + int numConnections = + connectionsPerNodeRecovery + + connectionsPerNodeBulk + + connectionsPerNodeReg + + connectionsPerNodeState + + connectionsPerNodeRecovery; + final ArrayList connections = new ArrayList<>(numConnections); + final InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address(); + for (int i = 0; i < numConnections; i++) { + connections.add(bootstrap.connect(address)); + } + final Iterator iterator = connections.iterator(); + try { + for (Channel[] channels : nodeChannels.getChannelArrays()) { + for (int i = 0; i < channels.length; i++) { + assert iterator.hasNext(); + ChannelFuture future = iterator.next(); + future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5)); + if (!future.isSuccess()) { + throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.cause()); + } + channels[i] = future.channel(); + channels[i].closeFuture().addListener(new ChannelCloseListener(node)); + } + } + if (nodeChannels.recovery.length == 0) { + if (nodeChannels.bulk.length > 0) { + nodeChannels.recovery = nodeChannels.bulk; + } else { + nodeChannels.recovery = nodeChannels.reg; + } + } + if (nodeChannels.bulk.length == 0) { + nodeChannels.bulk = nodeChannels.reg; + } + } catch (final RuntimeException e) { + for (final ChannelFuture future : Collections.unmodifiableList(connections)) { + FutureUtils.cancel(future); + if (future.channel() != null && future.channel().isOpen()) { + try { + future.channel().close(); + } catch (Exception inner) { + e.addSuppressed(inner); + } + } + } + throw e; + } + success = true; + } finally { + if (success == false) { + try { + nodeChannels.close(); + } catch (IOException e) { + logger.trace("exception while closing channels", e); + } + } + } + return nodeChannels; + } + + private class ChannelCloseListener implements ChannelFutureListener { + + private final DiscoveryNode node; + + private ChannelCloseListener(DiscoveryNode node) { + this.node = node; + } + + @Override + public void operationComplete(final ChannelFuture future) throws Exception { + NodeChannels nodeChannels = connectedNodes.get(node); + if (nodeChannels != null && nodeChannels.hasChannel(future.channel())) { + threadPool.generic().execute(() -> disconnectFromNode(node, future.channel(), "channel closed event")); + } + } + } + + @Override + protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) { + final ChannelFuture future = channel.writeAndFlush(Netty4Utils.toByteBuf(reference)); + if (close) { + future.addListener(f -> { + try { + sendListener.run(); + } finally { + future.channel().close(); + } + }); + } else { + future.addListener(f -> sendListener.run()); + } + } + + @Override + protected void closeChannels(final List channels) throws IOException { + Netty4Utils.closeChannels(channels); + } + + @Override + protected InetSocketAddress getLocalAddress(Channel channel) { + return (InetSocketAddress) channel.localAddress(); + } + + @Override + protected Channel bind(String name, InetSocketAddress address) { + return serverBootstraps.get(name).bind(address).syncUninterruptibly().channel(); + } + + ScheduledPing getPing() { + return scheduledPing; + } + + @Override + protected boolean isOpen(Channel channel) { + return channel.isOpen(); + } + + @Override + @SuppressForbidden(reason = "debug") + protected void stopInternal() { + Releasables.close(serverOpenChannels, () -> { + final List>> serverBootstrapCloseFutures = new ArrayList<>(serverBootstraps.size()); + for (final Map.Entry entry : serverBootstraps.entrySet()) { + serverBootstrapCloseFutures.add( + Tuple.tuple(entry.getKey(), entry.getValue().config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS))); + } + for (final Tuple> future : serverBootstrapCloseFutures) { + future.v2().awaitUninterruptibly(); + if (!future.v2().isSuccess()) { + logger.debug("Error closing server bootstrap for profile [{}]", future.v2().cause(), future.v1()); + } + } + serverBootstraps.clear(); + + if (bootstrap != null) { + bootstrap.config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS).awaitUninterruptibly(); + bootstrap = null; + } + }); + } + +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java new file mode 100644 index 00000000000..53cf1b329aa --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java @@ -0,0 +1,121 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.bytes.BytesReference; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class Netty4Utils { + + static { + InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() { + + @Override + public InternalLogger newInstance(final String name) { + return new Netty4InternalESLogger(name.replace("io.netty.", "netty.")); + } + + }); + } + + public static void setup() { + + } + + /** + * Turns the given BytesReference into a ByteBuf. Note: the returned ByteBuf will reference the internal + * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope. + */ + public static ByteBuf toByteBuf(final BytesReference reference) { + if (reference.length() == 0) { + return Unpooled.EMPTY_BUFFER; + } + if (reference instanceof ByteBufBytesReference) { + return ((ByteBufBytesReference) reference).toByteBuf(); + } else { + final BytesRefIterator iterator = reference.iterator(); + // usually we have one, two, or three components + // from the header, the message, and a buffer + final List buffers = new ArrayList<>(3); + try { + BytesRef slice; + while ((slice = iterator.next()) != null) { + buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length)); + } + final CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size()); + composite.addComponents(true, buffers); + return composite; + } catch (IOException ex) { + throw new AssertionError("no IO happens here", ex); + } + } + } + + /** + * Wraps the given ChannelBuffer with a BytesReference + */ + public static BytesReference toBytesReference(final ByteBuf buffer) { + return toBytesReference(buffer, buffer.readableBytes()); + } + + /** + * Wraps the given ChannelBuffer with a BytesReference of a given size + */ + static BytesReference toBytesReference(final ByteBuf buffer, final int size) { + return new ByteBufBytesReference(buffer, size); + } + + public static void closeChannels(final Collection channels) throws IOException { + IOException closingExceptions = null; + final List futures = new ArrayList<>(); + for (final Channel channel : channels) { + try { + if (channel != null && channel.isOpen()) { + futures.add(channel.close()); + } + } catch (Exception e) { + if (closingExceptions == null) { + closingExceptions = new IOException("failed to close channels"); + } + closingExceptions.addSuppressed(e); + } + } + for (final ChannelFuture future : futures) { + future.awaitUninterruptibly(); + } + + if (closingExceptions != null) { + throw closingExceptions; + } + } + +} diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..45c8cd923aa --- /dev/null +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // Netty SelectorUtil wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 + // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! + permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; +}; \ No newline at end of file diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java b/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java new file mode 100644 index 00000000000..1feded41ed5 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch; + +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.Netty4Plugin; +import org.elasticsearch.transport.netty4.Netty4Transport; + +import java.util.Collection; + +@ESIntegTestCase.SuppressLocalMode +public abstract class ESNetty4IntegTestCase extends ESIntegTestCase { + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected boolean addMockTransportService() { + return false; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + // randomize netty settings + if (randomBoolean()) { + builder.put(Netty4Transport.WORKER_COUNT.getKey(), random().nextInt(3) + 1); + } + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + builder.put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME); + return builder.build(); + } + + @Override + protected Settings transportClientSettings() { + Settings.Builder builder = Settings.builder().put(super.transportClientSettings()); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + return builder.build(); + } + + @Override + protected Collection> nodePlugins() { + return pluginList(Netty4Plugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return pluginList(Netty4Plugin.class); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java new file mode 100644 index 00000000000..58c6a476fe8 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java @@ -0,0 +1,490 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelConfig; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelId; +import io.netty.channel.ChannelMetadata; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ChannelProgressivePromise; +import io.netty.channel.ChannelPromise; +import io.netty.channel.EventLoop; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.util.Attribute; +import io.netty.util.AttributeKey; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.http.HttpTransportSettings; +import org.elasticsearch.http.netty4.cors.Netty4CorsHandler; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.netty4.Netty4Utils; +import org.junit.After; +import org.junit.Before; + +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class Netty4HttpChannelTests extends ESTestCase { + + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Settings.EMPTY); + threadPool = new TestThreadPool("test"); + bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + } + + public void testResponse() { + final FullHttpResponse response = executeRequest(Settings.EMPTY, "request-host"); + assertThat(response.content(), equalTo(Netty4Utils.toByteBuf(new TestResponse().content()))); + } + + public void testCorsEnabledWithoutAllowOrigins() { + // Set up a HTTP transport with only the CORS enabled setting + Settings settings = Settings.builder() + .put(HttpTransportSettings.SETTING_CORS_ENABLED.getKey(), true) + .build(); + HttpResponse response = executeRequest(settings, "remote-host", "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue()); + } + + public void testCorsEnabledWithAllowOrigins() { + final String originValue = "remote-host"; + // create a http transport with CORS enabled and allow origin configured + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue) + .build(); + HttpResponse response = executeRequest(settings, originValue, "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + } + + public void testCorsAllowOriginWithSameHost() { + String originValue = "remote-host"; + String host = "remote-host"; + // create a http transport with CORS enabled + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .build(); + HttpResponse response = executeRequest(settings, originValue, host); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + + originValue = "http://" + originValue; + response = executeRequest(settings, originValue, host); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + + originValue = originValue + ":5555"; + host = host + ":5555"; + response = executeRequest(settings, originValue, host); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + + originValue = originValue.replace("http", "https"); + response = executeRequest(settings, originValue, host); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + } + + public void testThatStringLiteralWorksOnMatch() { + final String originValue = "remote-host"; + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue) + .put(SETTING_CORS_ALLOW_METHODS.getKey(), "get, options, post") + .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true) + .build(); + HttpResponse response = executeRequest(settings, originValue, "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), equalTo("true")); + } + + public void testThatAnyOriginWorks() { + final String originValue = Netty4CorsHandler.ANY_ORIGIN; + Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue) + .build(); + HttpResponse response = executeRequest(settings, originValue, "request-host"); + // inspect response and validate + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue()); + String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN); + assertThat(allowedOrigins, is(originValue)); + assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), nullValue()); + } + + public void testHeadersSet() { + Settings settings = Settings.builder().build(); + try (Netty4HttpServerTransport httpServerTransport = + new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) { + httpServerTransport.start(); + final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); + httpRequest.headers().add(HttpHeaderNames.ORIGIN, "remote"); + final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel(); + Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel); + + // send a response + Netty4HttpChannel channel = + new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext()); + TestResponse resp = new TestResponse(); + final String customHeader = "custom-header"; + final String customHeaderValue = "xyz"; + resp.addHeader(customHeader, customHeaderValue); + channel.sendResponse(resp); + + // inspect what was written + List writtenObjects = writeCapturingChannel.getWrittenObjects(); + assertThat(writtenObjects.size(), is(1)); + HttpResponse response = (HttpResponse) writtenObjects.get(0); + assertThat(response.headers().get("non-existent-header"), nullValue()); + assertThat(response.headers().get(customHeader), equalTo(customHeaderValue)); + assertThat(response.headers().get(HttpHeaderNames.CONTENT_LENGTH), equalTo(Integer.toString(resp.content().length()))); + assertThat(response.headers().get(HttpHeaderNames.CONTENT_TYPE), equalTo(resp.contentType())); + } + } + + private FullHttpResponse executeRequest(final Settings settings, final String host) { + return executeRequest(settings, null, host); + } + + private FullHttpResponse executeRequest(final Settings settings, final String originValue, final String host) { + // construct request and send it over the transport layer + try (Netty4HttpServerTransport httpServerTransport = + new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) { + httpServerTransport.start(); + final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); + if (originValue != null) { + httpRequest.headers().add(HttpHeaderNames.ORIGIN, originValue); + } + httpRequest.headers().add(HttpHeaderNames.HOST, host); + final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel(); + final Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel); + + Netty4HttpChannel channel = + new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext()); + channel.sendResponse(new TestResponse()); + + // get the response + List writtenObjects = writeCapturingChannel.getWrittenObjects(); + assertThat(writtenObjects.size(), is(1)); + return (FullHttpResponse) writtenObjects.get(0); + } + } + + private static class WriteCapturingChannel implements Channel { + + private List writtenObjects = new ArrayList<>(); + + @Override + public ChannelId id() { + return null; + } + + @Override + public EventLoop eventLoop() { + return null; + } + + @Override + public Channel parent() { + return null; + } + + @Override + public ChannelConfig config() { + return null; + } + + @Override + public boolean isOpen() { + return false; + } + + @Override + public boolean isRegistered() { + return false; + } + + @Override + public boolean isActive() { + return false; + } + + @Override + public ChannelMetadata metadata() { + return null; + } + + @Override + public SocketAddress localAddress() { + return null; + } + + @Override + public SocketAddress remoteAddress() { + return null; + } + + @Override + public ChannelFuture closeFuture() { + return null; + } + + @Override + public boolean isWritable() { + return false; + } + + @Override + public long bytesBeforeUnwritable() { + return 0; + } + + @Override + public long bytesBeforeWritable() { + return 0; + } + + @Override + public Unsafe unsafe() { + return null; + } + + @Override + public ChannelPipeline pipeline() { + return null; + } + + @Override + public ByteBufAllocator alloc() { + return null; + } + + @Override + public Channel read() { + return null; + } + + @Override + public Channel flush() { + return null; + } + + @Override + public ChannelFuture bind(SocketAddress localAddress) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress) { + return null; + } + + @Override + public ChannelFuture disconnect() { + return null; + } + + @Override + public ChannelFuture close() { + return null; + } + + @Override + public ChannelFuture deregister() { + return null; + } + + @Override + public ChannelFuture bind(SocketAddress localAddress, ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture disconnect(ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture close(ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture deregister(ChannelPromise promise) { + return null; + } + + @Override + public ChannelFuture write(Object msg) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelFuture write(Object msg, ChannelPromise promise) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelFuture writeAndFlush(Object msg) { + writtenObjects.add(msg); + return null; + } + + @Override + public ChannelPromise newPromise() { + return null; + } + + @Override + public ChannelProgressivePromise newProgressivePromise() { + return null; + } + + @Override + public ChannelFuture newSucceededFuture() { + return null; + } + + @Override + public ChannelFuture newFailedFuture(Throwable cause) { + return null; + } + + @Override + public ChannelPromise voidPromise() { + return null; + } + + @Override + public Attribute attr(AttributeKey key) { + return null; + } + + @Override + public boolean hasAttr(AttributeKey key) { + return false; + } + + @Override + public int compareTo(Channel o) { + return 0; + } + + List getWrittenObjects() { + return writtenObjects; + } + + } + + private static class TestResponse extends RestResponse { + + @Override + public String contentType() { + return "text"; + } + + @Override + public BytesReference content() { + return Netty4Utils.toBytesReference(Unpooled.copiedBuffer("content", StandardCharsets.UTF_8)); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java new file mode 100644 index 00000000000..31e7a6112f9 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java @@ -0,0 +1,190 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.http.netty4; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpRequestEncoder; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseDecoder; +import io.netty.handler.codec.http.HttpVersion; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.io.Closeable; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static io.netty.handler.codec.http.HttpHeaderNames.HOST; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +/** + * Tiny helper to send http requests over netty. + */ +class Netty4HttpClient implements Closeable { + + static Collection returnHttpResponseBodies(Collection responses) { + List list = new ArrayList<>(responses.size()); + for (FullHttpResponse response : responses) { + list.add(response.content().toString(StandardCharsets.UTF_8)); + } + return list; + } + + static Collection returnOpaqueIds(Collection responses) { + List list = new ArrayList<>(responses.size()); + for (HttpResponse response : responses) { + list.add(response.headers().get("X-Opaque-Id")); + } + return list; + } + + private final Bootstrap clientBootstrap; + + Netty4HttpClient() { + clientBootstrap = new Bootstrap().channel(NioSocketChannel.class).group(new NioEventLoopGroup()); + } + + public Collection get(SocketAddress remoteAddress, String... uris) throws InterruptedException { + Collection requests = new ArrayList<>(uris.length); + for (int i = 0; i < uris.length; i++) { + final HttpRequest httpRequest = new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]); + httpRequest.headers().add(HOST, "localhost"); + httpRequest.headers().add("X-Opaque-ID", String.valueOf(i)); + requests.add(httpRequest); + } + return sendRequests(remoteAddress, requests); + } + + @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. + public final Collection post(SocketAddress remoteAddress, Tuple... urisAndBodies) + throws InterruptedException { + return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies); + } + + @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. + public final Collection put(SocketAddress remoteAddress, Tuple... urisAndBodies) + throws InterruptedException { + return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies); + } + + private Collection processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple... urisAndBodies) throws InterruptedException { + Collection requests = new ArrayList<>(urisAndBodies.length); + for (Tuple uriAndBody : urisAndBodies) { + ByteBuf content = Unpooled.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8); + HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content); + request.headers().add(HttpHeaderNames.HOST, "localhost"); + request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()); + requests.add(request); + } + return sendRequests(remoteAddress, requests); + } + + private synchronized Collection sendRequests( + final SocketAddress remoteAddress, + final Collection requests) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(requests.size()); + final Collection content = Collections.synchronizedList(new ArrayList<>(requests.size())); + + clientBootstrap.handler(new CountDownLatchHandler(latch, content)); + + ChannelFuture channelFuture = null; + try { + channelFuture = clientBootstrap.connect(remoteAddress); + channelFuture.sync(); + + for (HttpRequest request : requests) { + channelFuture.channel().writeAndFlush(request); + } + latch.await(); + + } finally { + if (channelFuture != null) { + channelFuture.channel().close().sync(); + } + } + + return content; + } + + @Override + public void close() { + clientBootstrap.config().group().shutdownGracefully().awaitUninterruptibly(); + } + + /** + * helper factory which adds returned data to a list and uses a count down latch to decide when done + */ + private static class CountDownLatchHandler extends ChannelInitializer { + + private final CountDownLatch latch; + private final Collection content; + + CountDownLatchHandler(final CountDownLatch latch, final Collection content) { + this.latch = latch; + this.content = content; + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + final int maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt(); + ch.pipeline().addLast(new HttpResponseDecoder()); + ch.pipeline().addLast(new HttpRequestEncoder()); + ch.pipeline().addLast(new HttpObjectAggregator(maxContentLength)); + ch.pipeline().addLast(new SimpleChannelInboundHandler() { + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception { + final FullHttpResponse response = (FullHttpResponse) msg; + content.add(response.copy()); + latch.countDown(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + super.exceptionCaught(ctx, cause); + latch.countDown(); + } + }); + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPublishPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPublishPortTests.java new file mode 100644 index 00000000000..03e09f28d25 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPublishPortTests.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.http.BindHttpException; +import org.elasticsearch.http.HttpTransportSettings; +import org.elasticsearch.test.ESTestCase; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; + +import static java.net.InetAddress.getByName; +import static java.util.Arrays.asList; +import static org.elasticsearch.http.netty4.Netty4HttpServerTransport.resolvePublishPort; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class Netty4HttpPublishPortTests extends ESTestCase { + + public void testHttpPublishPort() throws Exception { + int boundPort = randomIntBetween(9000, 9100); + int otherBoundPort = randomIntBetween(9200, 9300); + + int publishPort = resolvePublishPort(Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT.getKey(), 9080).build(), + randomAddresses(), getByName("127.0.0.2")); + assertThat("Publish port should be explicitly set to 9080", publishPort, equalTo(9080)); + + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("127.0.0.1")); + assertThat("Publish port should be derived from matched address", publishPort, equalTo(boundPort)); + + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("127.0.0.1", boundPort), address("127.0.0.2", boundPort)), + getByName("127.0.0.3")); + assertThat("Publish port should be derived from unique port of bound addresses", publishPort, equalTo(boundPort)); + + final BindHttpException e = + expectThrows(BindHttpException.class, + () -> resolvePublishPort( + Settings.EMPTY, + asList(address("127.0.0.1", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("127.0.0.3"))); + assertThat(e.getMessage(), containsString("Failed to auto-resolve http publish port")); + + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("0.0.0.0", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("127.0.0.1")); + assertThat("Publish port should be derived from matching wildcard address", publishPort, equalTo(boundPort)); + + if (NetworkUtils.SUPPORTS_V6) { + publishPort = resolvePublishPort(Settings.EMPTY, asList(address("0.0.0.0", boundPort), address("127.0.0.2", otherBoundPort)), + getByName("::1")); + assertThat("Publish port should be derived from matching wildcard address", publishPort, equalTo(boundPort)); + } + } + + private InetSocketTransportAddress address(String host, int port) throws UnknownHostException { + return new InetSocketTransportAddress(getByName(host), port); + } + + private InetSocketTransportAddress randomAddress() throws UnknownHostException { + return address("127.0.0." + randomIntBetween(1, 100), randomIntBetween(9200, 9300)); + } + + private List randomAddresses() throws UnknownHostException { + List addresses = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(1, 5); i++) { + addresses.add(randomAddress()); + } + return addresses; + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java new file mode 100644 index 00000000000..f1e2e922cbd --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -0,0 +1,131 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; + +import java.util.Collection; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; + +/** + * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass. + * + * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing + * a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason. + */ +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1, transportClientRatio = 0) +public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase { + + private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB); + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), LIMIT) + .build(); + } + + public void testLimitsInFlightRequests() throws Exception { + ensureGreen(); + + // we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit + int numRequests = LIMIT.bytesAsInt() / 100; + + StringBuilder bulkRequest = new StringBuilder(); + for (int i = 0; i < numRequests; i++) { + bulkRequest.append("{\"index\": {}}"); + bulkRequest.append(System.lineSeparator()); + bulkRequest.append("{ \"field\" : \"value\" }"); + bulkRequest.append(System.lineSeparator()); + } + + @SuppressWarnings("unchecked") + Tuple[] requests = new Tuple[150]; + for (int i = 0; i < requests.length; i++) { + requests[i] = Tuple.tuple("/index/type/_bulk", bulkRequest); + } + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress + ().boundAddresses()); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection singleResponse = nettyHttpClient.post(inetSocketTransportAddress.address(), requests[0]); + assertThat(singleResponse, hasSize(1)); + assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK); + + Collection multipleResponses = nettyHttpClient.post(inetSocketTransportAddress.address(), requests); + assertThat(multipleResponses, hasSize(requests.length)); + assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE); + } + } + + public void testDoesNotLimitExcludedRequests() throws Exception { + ensureGreen(); + + @SuppressWarnings("unchecked") + Tuple[] requestUris = new Tuple[1500]; + for (int i = 0; i < requestUris.length; i++) { + requestUris[i] = Tuple.tuple("/_cluster/settings", + "{ \"transient\": {\"indices.ttl.interval\": \"40s\" } }"); + } + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress + ().boundAddresses()); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.put(inetSocketTransportAddress.address(), requestUris); + assertThat(responses, hasSize(requestUris.length)); + assertAllInExpectedStatus(responses, HttpResponseStatus.OK); + } + } + + private void assertAtLeastOnceExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { + long countExpectedStatus = responses.stream().filter(r -> r.status().equals(expectedStatus)).count(); + assertThat("Expected at least one request with status [" + expectedStatus + "]", countExpectedStatus, greaterThan(0L)); + } + + private void assertAllInExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { + long countUnexpectedStatus = responses.stream().filter(r -> r.status().equals(expectedStatus) == false).count(); + assertThat("Expected all requests with status [" + expectedStatus + "] but [" + countUnexpectedStatus + + "] requests had a different one", countUnexpectedStatus, equalTo(0L)); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java new file mode 100644 index 00000000000..d8e61908011 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -0,0 +1,260 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.QueryStringDecoder; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasSize; + +/** + * This test just tests, if he pipelining works in general with out any connection the Elasticsearch handler + */ +public class Netty4HttpServerPipeliningTests extends ESTestCase { + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Settings.EMPTY); + threadPool = new TestThreadPool("test"); + bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + } + + public void testThatHttpPipeliningWorksWhenEnabled() throws Exception { + final Settings settings = Settings.builder() + .put("http.pipelining", true) + .put("http.port", "0") + .build(); + try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) { + httpServerTransport.start(); + final InetSocketTransportAddress transportAddress = + (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses()); + + final int numberOfRequests = randomIntBetween(4, 16); + final List requests = new ArrayList<>(numberOfRequests); + for (int i = 0; i < numberOfRequests; i++) { + if (rarely()) { + requests.add("/slow?sleep=" + scaledRandomIntBetween(500, 1000)); + } else { + requests.add("/" + i); + } + } + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{})); + Collection responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses); + assertThat(responseBodies, contains(requests.toArray())); + } + } + } + + public void testThatHttpPipeliningCanBeDisabled() throws Exception { + final Settings settings = Settings.builder() + .put("http.pipelining", false) + .put("http.port", "0") + .build(); + try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) { + httpServerTransport.start(); + final InetSocketTransportAddress transportAddress = + (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress().boundAddresses()); + + final int numberOfRequests = randomIntBetween(4, 16); + final int numberOfSlowRequests = scaledRandomIntBetween(1, numberOfRequests); + final List requests = new ArrayList<>(numberOfRequests); + for (int i = 0; i < numberOfRequests - numberOfSlowRequests; i++) { + requests.add("/" + i); + } + for (int i = 0; i < numberOfSlowRequests; i++) { + requests.add("/slow?sleep=" + sleep(i)); + } + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{})); + List responseBodies = new ArrayList<>(Netty4HttpClient.returnHttpResponseBodies(responses)); + // we cannot be sure about the order of the fast requests, but the slow ones should have to be last + assertThat(responseBodies, hasSize(numberOfRequests)); + for (int i = 0; i < numberOfSlowRequests; i++) { + assertThat(responseBodies.get(numberOfRequests - numberOfSlowRequests + i), equalTo("/slow?sleep=" + sleep(i))); + } + } + } + } + + + private int sleep(int index) { + return 500 + 100 * (index + 1); + } + + class CustomNettyHttpServerTransport extends Netty4HttpServerTransport { + + private final ExecutorService executorService = Executors.newCachedThreadPool(); + + CustomNettyHttpServerTransport(final Settings settings) { + super(settings, + Netty4HttpServerPipeliningTests.this.networkService, + Netty4HttpServerPipeliningTests.this.bigArrays, + Netty4HttpServerPipeliningTests.this.threadPool); + } + + @Override + public ChannelHandler configureServerChannelHandler() { + return new CustomHttpChannelHandler(this, executorService, Netty4HttpServerPipeliningTests.this.threadPool.getThreadContext()); + } + + @Override + protected void doClose() { + executorService.shutdown(); + super.doClose(); + } + + } + + private class CustomHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler { + + private final ExecutorService executorService; + + CustomHttpChannelHandler(Netty4HttpServerTransport transport, ExecutorService executorService, ThreadContext threadContext) { + super(transport, randomBoolean(), threadContext); + this.executorService = executorService; + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + super.initChannel(ch); + ch.pipeline().replace("handler", "handler", new PossiblySlowUpstreamHandler(executorService)); + } + + } + + class PossiblySlowUpstreamHandler extends SimpleChannelInboundHandler { + + private final ExecutorService executorService; + + PossiblySlowUpstreamHandler(ExecutorService executorService) { + this.executorService = executorService; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception { + executorService.submit(new PossiblySlowRunnable(ctx, msg)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.info("Caught exception", cause); + ctx.channel().close().sync(); + } + + } + + class PossiblySlowRunnable implements Runnable { + + private ChannelHandlerContext ctx; + private HttpPipelinedRequest pipelinedRequest; + private FullHttpRequest fullHttpRequest; + + PossiblySlowRunnable(ChannelHandlerContext ctx, Object msg) { + this.ctx = ctx; + if (msg instanceof HttpPipelinedRequest) { + this.pipelinedRequest = (HttpPipelinedRequest) msg; + } else if (msg instanceof FullHttpRequest) { + this.fullHttpRequest = (FullHttpRequest) msg; + } + } + + @Override + public void run() { + final String uri; + if (pipelinedRequest != null && pipelinedRequest.last() instanceof FullHttpRequest) { + uri = ((FullHttpRequest) pipelinedRequest.last()).uri(); + } else { + uri = fullHttpRequest.uri(); + } + + final ByteBuf buffer = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8); + + final DefaultFullHttpResponse httpResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buffer); + httpResponse.headers().add(HttpHeaderNames.CONTENT_LENGTH, buffer.readableBytes()); + + final QueryStringDecoder decoder = new QueryStringDecoder(uri); + + final int timeout = + uri.startsWith("/slow") && decoder.parameters().containsKey("sleep") ? + Integer.valueOf(decoder.parameters().get("sleep").get(0)) : 0; + if (timeout > 0) { + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + if (pipelinedRequest != null) { + ctx.writeAndFlush(pipelinedRequest.createHttpResponse(httpResponse, ctx.channel().newPromise())); + } else { + ctx.writeAndFlush(httpResponse); + } + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java new file mode 100644 index 00000000000..dc63907265c --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.HttpMethod; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.http.netty4.cors.Netty4CorsConfig; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests for the {@link Netty4HttpServerTransport} class. + */ +public class Netty4HttpServerTransportTests extends ESTestCase { + + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Settings.EMPTY); + threadPool = new TestThreadPool("test"); + bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + threadPool = null; + networkService = null; + bigArrays = null; + } + + public void testCorsConfig() { + final Set methods = new HashSet<>(Arrays.asList("get", "options", "post")); + final Set headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length")); + final Settings settings = Settings.builder() + .put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*") + .put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToCommaDelimitedString(methods)) + .put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToCommaDelimitedString(headers)) + .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true) + .build(); + final Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool); + final Netty4CorsConfig corsConfig = transport.getCorsConfig(); + assertThat(corsConfig.isAnyOriginSupported(), equalTo(true)); + assertThat(corsConfig.allowedRequestHeaders(), equalTo(headers)); + assertThat(corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()), equalTo(methods)); + transport.close(); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningDisabledIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningDisabledIT.java new file mode 100644 index 00000000000..4ea46c651fc --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningDisabledIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4PipeliningDisabledIT extends ESNetty4IntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put("http.pipelining", false) + .build(); + } + + public void testThatNettyHttpServerDoesNotSupportPipelining() throws Exception { + ensureGreen(); + String[] requests = new String[] {"/", "/_nodes/stats", "/", "/_cluster/state", "/", "/_nodes", "/"}; + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(boundAddresses); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(inetSocketTransportAddress.address(), requests); + assertThat(responses, hasSize(requests.length)); + + List opaqueIds = new ArrayList<>(Netty4HttpClient.returnOpaqueIds(responses)); + + assertResponsesOutOfOrder(opaqueIds); + } + } + + /** + * checks if all responses are there, but also tests that they are out of order because pipelining is disabled + */ + private void assertResponsesOutOfOrder(List opaqueIds) { + String message = String.format(Locale.ROOT, "Expected returned http message ids to be in any order of: %s", opaqueIds); + assertThat(message, opaqueIds, containsInAnyOrder("0", "1", "2", "3", "4", "5", "6")); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningEnabledIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningEnabledIT.java new file mode 100644 index 00000000000..c2222562d72 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningEnabledIT.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; + +import java.util.Collection; +import java.util.Locale; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4PipeliningEnabledIT extends ESNetty4IntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put("http.pipelining", true) + .build(); + } + + public void testThatNettyHttpServerSupportsPipelining() throws Exception { + String[] requests = new String[]{"/", "/_nodes/stats", "/", "/_cluster/state", "/"}; + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(boundAddresses); + + try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) { + Collection responses = nettyHttpClient.get(inetSocketTransportAddress.address(), requests); + assertThat(responses, hasSize(5)); + + Collection opaqueIds = Netty4HttpClient.returnOpaqueIds(responses); + assertOpaqueIdsInOrder(opaqueIds); + } + } + + private void assertOpaqueIdsInOrder(Collection opaqueIds) { + // check if opaque ids are monotonically increasing + int i = 0; + String msg = String.format(Locale.ROOT, "Expected list of opaque ids to be monotonically increasing, got [%s]", opaqueIds); + for (String opaqueId : opaqueIds) { + assertThat(msg, opaqueId, is(String.valueOf(i++))); + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4RestIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4RestIT.java new file mode 100644 index 00000000000..71cbf70abb3 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4RestIT.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; +import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +//TODO: This is a *temporary* workaround to ensure a timeout does not mask other problems +@TimeoutSuite(millis = 30 * TimeUnits.MINUTE) +public class Netty4RestIT extends ESClientYamlSuiteTestCase { + + public Netty4RestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESClientYamlSuiteTestCase.createParameters(0, 1); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java new file mode 100644 index 00000000000..1feb92223a3 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/pipelining/Netty4HttpPipeliningHandlerTests.java @@ -0,0 +1,262 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http.netty4.pipelining; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.QueryStringDecoder; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static org.hamcrest.core.Is.is; + +public class Netty4HttpPipeliningHandlerTests extends ESTestCase { + + private ExecutorService executorService = Executors.newFixedThreadPool(randomIntBetween(4, 8)); + private Map waitingRequests = new ConcurrentHashMap<>(); + private Map finishingRequests = new ConcurrentHashMap<>(); + + @After + public void tearDown() throws Exception { + waitingRequests.keySet().forEach(this::finishRequest); + shutdownExecutorService(); + super.tearDown(); + } + + private CountDownLatch finishRequest(String url) { + waitingRequests.get(url).countDown(); + return finishingRequests.get(url); + } + + private void shutdownExecutorService() throws InterruptedException { + if (!executorService.isShutdown()) { + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + } + } + + public void testThatPipeliningWorksWithFastSerializedRequests() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler()); + + for (int i = 0; i < numberOfRequests; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i))); + } + + final List latches = new ArrayList<>(); + for (final String url : waitingRequests.keySet()) { + latches.add(finishRequest(url)); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + embeddedChannel.flush(); + + for (int i = 0; i < numberOfRequests; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + + assertTrue(embeddedChannel.isOpen()); + } + + public void testThatPipeliningWorksWhenSlowRequestsInDifferentOrder() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler()); + + for (int i = 0; i < numberOfRequests; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i))); + } + + // random order execution + final List urls = new ArrayList<>(waitingRequests.keySet()); + Randomness.shuffle(urls); + final List latches = new ArrayList<>(); + for (final String url : urls) { + latches.add(finishRequest(url)); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + embeddedChannel.flush(); + + for (int i = 0; i < numberOfRequests; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + + assertTrue(embeddedChannel.isOpen()); + } + + public void testThatPipeliningWorksWithChunkedRequests() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = + new EmbeddedChannel( + new AggregateUrisAndHeadersHandler(), + new HttpPipeliningHandler(numberOfRequests), + new WorkEmulatorHandler()); + + for (int i = 0; i < numberOfRequests; i++) { + final DefaultHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/" + i); + embeddedChannel.writeInbound(request); + embeddedChannel.writeInbound(LastHttpContent.EMPTY_LAST_CONTENT); + } + + final List latches = new ArrayList<>(); + for (int i = numberOfRequests - 1; i >= 0; i--) { + latches.add(finishRequest(Integer.toString(i))); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + embeddedChannel.flush(); + + for (int i = 0; i < numberOfRequests; i++) { + assertReadHttpMessageHasContent(embeddedChannel, Integer.toString(i)); + } + + assertTrue(embeddedChannel.isOpen()); + } + + public void testThatPipeliningClosesConnectionWithTooManyEvents() throws InterruptedException { + final int numberOfRequests = randomIntBetween(2, 128); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(numberOfRequests), new WorkEmulatorHandler()); + + for (int i = 0; i < 1 + numberOfRequests + 1; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + Integer.toString(i))); + } + + final List latches = new ArrayList<>(); + final List requests = IntStream.range(1, numberOfRequests + 1).mapToObj(r -> r).collect(Collectors.toList()); + Randomness.shuffle(requests); + + for (final Integer request : requests) { + latches.add(finishRequest(request.toString())); + } + + for (final CountDownLatch latch : latches) { + latch.await(); + } + + finishRequest(Integer.toString(numberOfRequests + 1)).await(); + + embeddedChannel.flush(); + + assertFalse(embeddedChannel.isOpen()); + } + + + private void assertReadHttpMessageHasContent(EmbeddedChannel embeddedChannel, String expectedContent) { + FullHttpResponse response = (FullHttpResponse) embeddedChannel.outboundMessages().poll(); + assertNotNull("Expected response to exist, maybe you did not wait long enough?", response); + assertNotNull("Expected response to have content " + expectedContent, response.content()); + String data = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8); + assertThat(data, is(expectedContent)); + } + + private FullHttpRequest createHttpRequest(String uri) { + return new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uri); + } + + private static class AggregateUrisAndHeadersHandler extends SimpleChannelInboundHandler { + + static final Queue QUEUE_URI = new LinkedTransferQueue<>(); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) throws Exception { + QUEUE_URI.add(request.uri()); + } + + } + + private class WorkEmulatorHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(final ChannelHandlerContext ctx, final HttpPipelinedRequest pipelinedRequest) throws Exception { + final QueryStringDecoder decoder; + if (pipelinedRequest.last() instanceof FullHttpRequest) { + final FullHttpRequest fullHttpRequest = (FullHttpRequest) pipelinedRequest.last(); + decoder = new QueryStringDecoder(fullHttpRequest.uri()); + } else { + decoder = new QueryStringDecoder(AggregateUrisAndHeadersHandler.QUEUE_URI.poll()); + } + + final String uri = decoder.path().replace("/", ""); + final ByteBuf content = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8); + final DefaultFullHttpResponse httpResponse = new DefaultFullHttpResponse(HTTP_1_1, OK, content); + httpResponse.headers().add(CONTENT_LENGTH, content.readableBytes()); + + final CountDownLatch waitingLatch = new CountDownLatch(1); + waitingRequests.put(uri, waitingLatch); + final CountDownLatch finishingLatch = new CountDownLatch(1); + finishingRequests.put(uri, finishingLatch); + + executorService.submit(() -> { + try { + waitingLatch.await(1000, TimeUnit.SECONDS); + final ChannelPromise promise = ctx.newPromise(); + ctx.write(pipelinedRequest.createHttpResponse(httpResponse, promise), promise); + finishingLatch.countDown(); + } catch (InterruptedException e) { + fail(e.toString()); + } + }); + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java new file mode 100644 index 00000000000..bce875e8516 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; + +import java.io.IOException; + +public class ByteBufBytesReferenceTests extends AbstractBytesReferenceTestCase { + @Override + protected BytesReference newBytesReference(int length) throws IOException { + ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays); + for (int i = 0; i < length; i++) { + out.writeByte((byte) random().nextInt(1 << 8)); + } + assertEquals(out.size(), length); + BytesReference ref = out.bytes(); + assertEquals(ref.length(), length); + BytesRef bytesRef = ref.toBytesRef(); + final ByteBuf buffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length); + return Netty4Utils.toBytesReference(buffer); + } + + public void testSliceOnAdvancedBuffer() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); + BytesRef bytesRef = bytesReference.toBytesRef(); + ByteBuf channelBuffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); + int numBytesToRead = randomIntBetween(1, 5); + for (int i = 0; i < numBytesToRead; i++) { + channelBuffer.readByte(); + } + BytesReference other = Netty4Utils.toBytesReference(channelBuffer); + BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead); + assertEquals(other, slice); + assertEquals(other.slice(3, 1), slice.slice(3, 1)); + } + + public void testImmutable() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); + BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef()); + ByteBuf channelBuffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); + ByteBufBytesReference byteBufBytesReference = new ByteBufBytesReference(channelBuffer, bytesRef.length); + assertEquals(byteBufBytesReference, bytesReference); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(byteBufBytesReference, bytesReference); + assertEquals(bytesRef, byteBufBytesReference.toBytesRef()); + + BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100)); + channelBuffer = Unpooled.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length); + byteBufBytesReference = new ByteBufBytesReference(channelBuffer, unicodeBytes.length); + String utf8ToString = byteBufBytesReference.utf8ToString(); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(utf8ToString, byteBufBytesReference.utf8ToString()); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java new file mode 100644 index 00000000000..0e2ee23f950 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestHandler; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponseHandler; +import org.elasticsearch.transport.TransportResponseOptions; +import org.elasticsearch.transport.TransportSettings; + +import java.io.IOException; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class Netty4ScheduledPingTests extends ESTestCase { + public void testScheduledPing() throws Exception { + ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + Settings settings = Settings.builder() + .put(TcpTransport.PING_SCHEDULE.getKey(), "5ms") + .put(TransportSettings.PORT.getKey(), 0) + .put("cluster.name", "test") + .build(); + + CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + + NamedWriteableRegistry registryA = new NamedWriteableRegistry(); + final Netty4Transport nettyA = new Netty4Transport(settings, threadPool, new NetworkService(settings), + BigArrays.NON_RECYCLING_INSTANCE, registryA, circuitBreakerService); + MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool); + serviceA.start(); + serviceA.acceptIncomingRequests(); + + NamedWriteableRegistry registryB = new NamedWriteableRegistry(); + final Netty4Transport nettyB = new Netty4Transport(settings, threadPool, new NetworkService(settings), + BigArrays.NON_RECYCLING_INSTANCE, registryB, circuitBreakerService); + MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool); + + serviceB.start(); + serviceB.acceptIncomingRequests(); + + DiscoveryNode nodeA = + new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode nodeB = + new DiscoveryNode("TS_B", "TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), Version.CURRENT); + + serviceA.connectToNode(nodeB); + serviceB.connectToNode(nodeA); + + assertBusy(new Runnable() { + @Override + public void run() { + assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L)); + assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L)); + } + }); + assertThat(nettyA.getPing().getFailedPings(), equalTo(0L)); + assertThat(nettyB.getPing().getFailedPings(), equalTo(0L)); + + serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC, + new TransportRequestHandler() { + @Override + public void messageReceived(TransportRequest.Empty request, TransportChannel channel) { + try { + channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY); + } catch (IOException e) { + logger.error("Unexpected failure", e); + fail(e.getMessage()); + } + } + }); + + int rounds = scaledRandomIntBetween(100, 5000); + for (int i = 0; i < rounds; i++) { + serviceB.submitRequest(nodeA, "sayHello", + TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(randomBoolean()).build(), + new TransportResponseHandler() { + @Override + public TransportResponse.Empty newInstance() { + return TransportResponse.Empty.INSTANCE; + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + public void handleResponse(TransportResponse.Empty response) { + } + + @Override + public void handleException(TransportException exp) { + logger.error("Unexpected failure", exp); + fail("got exception instead of a response: " + exp.getMessage()); + } + }).txGet(); + } + + assertBusy(() -> { + assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(200L)); + assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(200L)); + }); + assertThat(nettyA.getPing().getFailedPings(), equalTo(0L)); + assertThat(nettyB.getPing().getFailedPings(), equalTo(0L)); + + Releasables.close(serviceA, serviceB); + terminate(threadPool); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java new file mode 100644 index 00000000000..8e7ac400f46 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportSettings; +import org.junit.After; +import org.junit.Before; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.is; + +/** + * This test checks, if a HTTP look-alike request (starting with a HTTP method and a space) + * actually returns text response instead of just dropping the connection + */ +public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase { + + private final Settings settings = Settings.builder() + .put("node.name", "NettySizeHeaderFrameDecoderTests") + .put(TransportSettings.BIND_HOST.getKey(), "127.0.0.1") + .put(TransportSettings.PORT.getKey(), "0") + .build(); + + private ThreadPool threadPool; + private Netty4Transport nettyTransport; + private int port; + private InetAddress host; + + @Before + public void startThreadPool() { + threadPool = new ThreadPool(settings); + NetworkService networkService = new NetworkService(settings); + BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + nettyTransport = new Netty4Transport(settings, threadPool, networkService, bigArrays, new NamedWriteableRegistry(), + new NoneCircuitBreakerService()); + nettyTransport.start(); + + TransportAddress[] boundAddresses = nettyTransport.boundAddress().boundAddresses(); + InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) randomFrom(boundAddresses); + port = transportAddress.address().getPort(); + host = transportAddress.address().getAddress(); + } + + @After + public void terminateThreadPool() throws InterruptedException { + nettyTransport.stop(); + terminate(threadPool); + threadPool = null; + } + + public void testThatTextMessageIsReturnedOnHTTPLikeRequest() throws Exception { + String randomMethod = randomFrom("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH"); + String data = randomMethod + " / HTTP/1.1"; + + try (Socket socket = new Socket(host, port)) { + socket.getOutputStream().write(data.getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { + assertThat(reader.readLine(), is("This is not a HTTP port")); + } + } + } + + public void testThatNothingIsReturnedForOtherInvalidPackets() throws Exception { + try (Socket socket = new Socket(host, port)) { + socket.getOutputStream().write("FOOBAR".getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + + // end of stream + assertThat(socket.getInputStream().read(), is(-1)); + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java new file mode 100644 index 00000000000..b913fad933f --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import io.netty.channel.Channel; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportSettings; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4TransportIT extends ESNetty4IntegTestCase { + // static so we can use it in anonymous classes + private static String channelProfileName = null; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.TRANSPORT_TYPE_KEY, "exception-throwing").build(); + } + + @Override + protected Collection> nodePlugins() { + List> list = new ArrayList<>(); + list.add(ExceptionThrowingNetty4Transport.TestPlugin.class); + list.addAll(super.nodePlugins()); + return Collections.unmodifiableCollection(list); + } + + public void testThatConnectionFailsAsIntended() throws Exception { + Client transportClient = internalCluster().transportClient(); + ClusterHealthResponse clusterIndexHealths = transportClient.admin().cluster().prepareHealth().get(); + assertThat(clusterIndexHealths.getStatus(), is(ClusterHealthStatus.GREEN)); + try { + transportClient.filterWithHeader(Collections.singletonMap("ERROR", "MY MESSAGE")).admin().cluster().prepareHealth().get(); + fail("Expected exception, but didn't happen"); + } catch (ElasticsearchException e) { + assertThat(e.getMessage(), containsString("MY MESSAGE")); + assertThat(channelProfileName, is(TransportSettings.DEFAULT_PROFILE)); + } + } + + public static final class ExceptionThrowingNetty4Transport extends Netty4Transport { + + public static class TestPlugin extends Plugin { + public void onModule(NetworkModule module) { + module.registerTransport("exception-throwing", ExceptionThrowingNetty4Transport.class); + } + } + + @Inject + public ExceptionThrowingNetty4Transport( + Settings settings, + ThreadPool threadPool, + NetworkService networkService, + BigArrays bigArrays, + NamedWriteableRegistry namedWriteableRegistry, + CircuitBreakerService circuitBreakerService) { + super(settings, threadPool, networkService, bigArrays, namedWriteableRegistry, circuitBreakerService); + } + + protected String handleRequest(Channel channel, String profileName, + StreamInput stream, long requestId, int messageLengthBytes, Version version, + InetSocketAddress remoteAddress) throws IOException { + String action = super.handleRequest(channel, profileName, stream, requestId, messageLengthBytes, version, + remoteAddress); + channelProfileName = TransportSettings.DEFAULT_PROFILE; + return action; + } + + @Override + protected void validateRequest(StreamInput buffer, long requestId, String action) + throws IOException { + super.validateRequest(buffer, requestId, action); + String error = threadPool.getThreadContext().getHeader("ERROR"); + if (error != null) { + throw new ElasticsearchException(error); + } + } + + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java new file mode 100644 index 00000000000..32268d1f5d8 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.junit.annotations.Network; +import org.elasticsearch.transport.MockTransportClient; +import org.elasticsearch.transport.Netty4Plugin; + +import java.net.InetAddress; +import java.util.Locale; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) +public class Netty4TransportMultiPortIntegrationIT extends ESNetty4IntegTestCase { + + private static int randomPort = -1; + private static String randomPortRange; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + if (randomPort == -1) { + randomPort = randomIntBetween(49152, 65525); + randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort+10); + } + Settings.Builder builder = Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("network.host", "127.0.0.1") + .put("transport.profiles.client1.port", randomPortRange) + .put("transport.profiles.client1.publish_host", "127.0.0.7") + .put("transport.profiles.client1.publish_port", "4321") + .put("transport.profiles.client1.reuse_address", true); + return builder.build(); + } + + public void testThatTransportClientCanConnect() throws Exception { + Settings settings = Settings.builder() + .put("cluster.name", internalCluster().getClusterName()) + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + try (TransportClient transportClient = new MockTransportClient(settings, Netty4Plugin.class)) { + transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort)); + ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get(); + assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN)); + } + } + + @Network + public void testThatInfosAreExposed() throws Exception { + NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setTransport(true).get(); + for (NodeInfo nodeInfo : response.getNodes()) { + assertThat(nodeInfo.getTransport().getProfileAddresses().keySet(), hasSize(1)); + assertThat(nodeInfo.getTransport().getProfileAddresses(), hasKey("client1")); + BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getProfileAddresses().get("client1"); + for (TransportAddress transportAddress : boundTransportAddress.boundAddresses()) { + assertThat(transportAddress, instanceOf(InetSocketTransportAddress.class)); + } + + // bound addresses + for (TransportAddress transportAddress : boundTransportAddress.boundAddresses()) { + assertThat(transportAddress, instanceOf(InetSocketTransportAddress.class)); + assertThat(((InetSocketTransportAddress) transportAddress).address().getPort(), + is(allOf(greaterThanOrEqualTo(randomPort), lessThanOrEqualTo(randomPort + 10)))); + } + + // publish address + assertThat(boundTransportAddress.publishAddress(), instanceOf(InetSocketTransportAddress.class)); + InetSocketTransportAddress publishAddress = (InetSocketTransportAddress) boundTransportAddress.publishAddress(); + assertThat(NetworkAddress.format(publishAddress.address().getAddress()), is("127.0.0.7")); + assertThat(publishAddress.address().getPort(), is(4321)); + } + } +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java new file mode 100644 index 00000000000..0b8e4ed85b5 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.node.Node; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.Netty4Plugin; + +import java.net.Inet4Address; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; + +/** + * Checks that Elasticsearch produces a sane publish_address when it binds to + * different ports on ipv4 and ipv6. + */ +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class Netty4TransportPublishAddressIT extends ESNetty4IntegTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) + .build(); + } + + public void testDifferentPorts() throws Exception { + if (!NetworkUtils.SUPPORTS_V6) { + return; + } + logger.info("--> starting a node on ipv4 only"); + Settings ipv4Settings = Settings.builder().put("network.host", "127.0.0.1").build(); + String ipv4OnlyNode = internalCluster().startNode(ipv4Settings); // should bind 127.0.0.1:XYZ + + logger.info("--> starting a node on ipv4 and ipv6"); + Settings bothSettings = Settings.builder().put("network.host", "_local_").build(); + internalCluster().startNode(bothSettings); // should bind [::1]:XYZ and 127.0.0.1:XYZ+1 + + logger.info("--> waiting for the cluster to declare itself stable"); + ensureStableCluster(2); // fails if port of publish address does not match corresponding bound address + + logger.info("--> checking if boundAddress matching publishAddress has same port"); + NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get(); + for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) { + BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getAddress(); + if (nodeInfo.getNode().getName().equals(ipv4OnlyNode)) { + assertThat(boundTransportAddress.boundAddresses().length, equalTo(1)); + assertThat(boundTransportAddress.boundAddresses()[0].getPort(), equalTo(boundTransportAddress.publishAddress().getPort())); + } else { + assertThat(boundTransportAddress.boundAddresses().length, greaterThan(1)); + for (TransportAddress boundAddress : boundTransportAddress.boundAddresses()) { + assertThat(boundAddress, instanceOf(InetSocketTransportAddress.class)); + InetSocketTransportAddress inetBoundAddress = (InetSocketTransportAddress) boundAddress; + if (inetBoundAddress.address().getAddress() instanceof Inet4Address) { + // IPv4 address is preferred publish address for _local_ + assertThat(inetBoundAddress.getPort(), equalTo(boundTransportAddress.publishAddress().getPort())); + } + } + } + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java new file mode 100644 index 00000000000..8372a8540b8 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class Netty4UtilsTests extends ESTestCase { + + private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE; + private final BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false); + + public void testToChannelBufferWithEmptyRef() throws IOException { + ByteBuf buffer = Netty4Utils.toByteBuf(getRandomizedBytesReference(0)); + assertSame(Unpooled.EMPTY_BUFFER, buffer); + } + + public void testToChannelBufferWithSlice() throws IOException { + BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE)); + int sliceOffset = randomIntBetween(0, ref.length()); + int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset); + BytesReference slice = ref.slice(sliceOffset, sliceLength); + ByteBuf buffer = Netty4Utils.toByteBuf(slice); + BytesReference bytesReference = Netty4Utils.toBytesReference(buffer); + assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(bytesReference)); + } + + public void testToChannelBufferWithSliceAfter() throws IOException { + BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE)); + int sliceOffset = randomIntBetween(0, ref.length()); + int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset); + ByteBuf buffer = Netty4Utils.toByteBuf(ref); + BytesReference bytesReference = Netty4Utils.toBytesReference(buffer); + assertArrayEquals(BytesReference.toBytes(ref.slice(sliceOffset, sliceLength)), + BytesReference.toBytes(bytesReference.slice(sliceOffset, sliceLength))); + } + + public void testToChannelBuffer() throws IOException { + BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE)); + ByteBuf buffer = Netty4Utils.toByteBuf(ref); + BytesReference bytesReference = Netty4Utils.toBytesReference(buffer); + if (ref instanceof ByteBufBytesReference) { + assertEquals(buffer, ((ByteBufBytesReference) ref).toByteBuf()); + } else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer + assertTrue(buffer instanceof CompositeByteBuf); + } + assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference)); + } + + private BytesReference getRandomizedBytesReference(int length) throws IOException { + // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content + ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays); + for (int i = 0; i < length; i++) { + out.writeByte((byte) random().nextInt(1 << 8)); + } + assertEquals(out.size(), length); + BytesReference ref = out.bytes(); + assertEquals(ref.length(), length); + if (randomBoolean()) { + return new BytesArray(ref.toBytesRef()); + } else if (randomBoolean()) { + BytesRef bytesRef = ref.toBytesRef(); + return Netty4Utils.toBytesReference(Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length)); + } else { + return ref; + } + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java new file mode 100644 index 00000000000..e76431daf8d --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.TransportSettings; +import org.junit.Before; + +import static org.hamcrest.Matchers.is; + +public class NettyTransportMultiPortTests extends ESTestCase { + + private String host; + + @Before + public void setup() { + if (NetworkUtils.SUPPORTS_V6 && randomBoolean()) { + host = "::1"; + } else { + host = "127.0.0.1"; + } + } + + public void testThatNettyCanBindToMultiplePorts() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 22) // will not actually bind to this + .put("transport.profiles.default.port", 0) + .put("transport.profiles.client1.port", 0) + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(1, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatDefaultProfileInheritsFromStandardSettings() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 0) + .put("transport.profiles.client1.port", 0) + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(1, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatProfileWithoutPortSettingsFails() throws Exception { + + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 0) + .put("transport.profiles.client1.whatever", "foo") + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatDefaultProfilePortOverridesGeneralConfiguration() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 22) // will not actually bind to this + .put("transport.profiles.default.port", 0) + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + public void testThatProfileWithoutValidNameIsIgnored() throws Exception { + Settings settings = Settings.builder() + .put("network.host", host) + .put(TransportSettings.PORT.getKey(), 0) + // mimics someone trying to define a profile for .local which is the profile for a node request to itself + .put("transport.profiles." + TransportService.DIRECT_RESPONSE_PROFILE + ".port", 22) // will not actually bind to this + .put("transport.profiles..port", 23) // will not actually bind to this + .build(); + + ThreadPool threadPool = new TestThreadPool("tst"); + try (TcpTransport transport = startTransport(settings, threadPool)) { + assertEquals(0, transport.profileBoundAddresses().size()); + assertEquals(1, transport.boundAddress().boundAddresses().length); + } finally { + terminate(threadPool); + } + } + + private TcpTransport startTransport(Settings settings, ThreadPool threadPool) { + BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); + TcpTransport transport = new Netty4Transport(settings, threadPool, new NetworkService(settings), bigArrays, + new NamedWriteableRegistry(), new NoneCircuitBreakerService()); + transport.start(); + + assertThat(transport.lifecycleState(), is(Lifecycle.State.STARTED)); + return transport; + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java new file mode 100644 index 00000000000..bafec9d2ffa --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport.netty4; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.AbstractSimpleTransportTestCase; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportSettings; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.containsString; + +public class SimpleNetty4TransportTests extends AbstractSimpleTransportTestCase { + + public static MockTransportService nettyFromThreadPool( + Settings settings, + ThreadPool threadPool, final Version version) { + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); + Transport transport = new Netty4Transport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE, + namedWriteableRegistry, new NoneCircuitBreakerService()) { + @Override + protected Version getCurrentVersion() { + return version; + } + }; + return new MockTransportService(Settings.EMPTY, transport, threadPool); + } + + @Override + protected MockTransportService build(Settings settings, Version version) { + settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build(); + MockTransportService transportService = nettyFromThreadPool(settings, threadPool, version); + transportService.start(); + return transportService; + } + + public void testConnectException() throws UnknownHostException { + try { + serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876), + emptyMap(), emptySet(),Version.CURRENT)); + fail("Expected ConnectTransportException"); + } catch (ConnectTransportException e) { + assertThat(e.getMessage(), containsString("connect_timeout")); + assertThat(e.getMessage(), containsString("[127.0.0.1:9876]")); + } + } + +} diff --git a/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml b/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml new file mode 100644 index 00000000000..a7beac1491f --- /dev/null +++ b/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yaml @@ -0,0 +1,13 @@ +# Integration tests for Netty transport +# +"Netty loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.modules.1.name: transport-netty4 } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java index 8da56d5a72b..17e5e1c3ca7 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AnalysisICURestIT extends ESRestTestCase { +public class AnalysisICURestIT extends ESClientYamlSuiteTestCase { public AnalysisICURestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AnalysisICURestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java index ae51e491d6b..5697fb55bde 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AnalysisKuromojiRestIT extends ESRestTestCase { +public class AnalysisKuromojiRestIT extends ESClientYamlSuiteTestCase { public AnalysisKuromojiRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AnalysisKuromojiRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java index 9d66bf24357..48f1c4b1716 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AnalysisPhoneticRestIT extends ESRestTestCase { +public class AnalysisPhoneticRestIT extends ESClientYamlSuiteTestCase { public AnalysisPhoneticRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AnalysisPhoneticRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java index 16113b2b7ac..58c86acea84 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AnalysisSmartChineseRestIT extends ESRestTestCase { +public class AnalysisSmartChineseRestIT extends ESClientYamlSuiteTestCase { public AnalysisSmartChineseRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AnalysisSmartChineseRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java index 330ad87af74..54da5de8e4a 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AnalysisPolishRestIT extends ESRestTestCase { +public class AnalysisPolishRestIT extends ESClientYamlSuiteTestCase { public AnalysisPolishRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AnalysisPolishRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/plugins/discovery-azure-classic/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java index cb04842cb4a..5506169f79d 100644 --- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java +++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.discovery.azure.classic; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AzureDiscoveryRestIT extends ESRestTestCase { +public class AzureDiscoveryRestIT extends ESClientYamlSuiteTestCase { public AzureDiscoveryRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AzureDiscoveryRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 15cce1aa32b..a31dcde20a0 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -44,7 +44,11 @@ dependencyLicenses { test { // this is needed for insecure plugins, remove if possible! - systemProperty 'tests.artifact', project.name + systemProperty 'tests.artifact', project.name + // this could be needed by AwsEc2ServiceImplTests#testAWSCredentialsWithSystemProviders() + // As it's marked as Ignored for now, we can comment those + // systemProperty 'aws.accessKeyId', 'DUMMY_ACCESS_KEY' + // systemProperty 'aws.secretKey', 'DUMMY_SECRET_KEY' } thirdPartyAudit.excludes = [ diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java index 649f84b7aed..b51c9c03f6e 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; @@ -66,15 +67,45 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements Aws return client; } + this.client = new AmazonEC2Client(buildCredentials(logger, settings), buildConfiguration(logger, settings)); + String endpoint = findEndpoint(logger, settings); + if (endpoint != null) { + client.setEndpoint(endpoint); + } + + return this.client; + } + + protected static AWSCredentialsProvider buildCredentials(ESLogger logger, Settings settings) { + AWSCredentialsProvider credentials; + + String key = CLOUD_EC2.KEY_SETTING.get(settings); + String secret = CLOUD_EC2.SECRET_SETTING.get(settings); + if (key.isEmpty() && secret.isEmpty()) { + logger.debug("Using either environment variables, system properties or instance profile credentials"); + credentials = new AWSCredentialsProviderChain( + new EnvironmentVariableCredentialsProvider(), + new SystemPropertiesCredentialsProvider(), + new InstanceProfileCredentialsProvider() + ); + } else { + logger.debug("Using basic key/secret credentials"); + credentials = new AWSCredentialsProviderChain( + new StaticCredentialsProvider(new BasicAWSCredentials(key, secret)) + ); + } + + return credentials; + } + + protected static ClientConfiguration buildConfiguration(ESLogger logger, Settings settings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings)); - String key = CLOUD_EC2.KEY_SETTING.get(settings); - String secret = CLOUD_EC2.SECRET_SETTING.get(settings); - if (CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) { + if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) { String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings); Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings); String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings); @@ -97,78 +128,86 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements Aws // Increase the number of retries in case of 5xx API responses final Random rand = Randomness.get(); RetryPolicy retryPolicy = new RetryPolicy( - RetryPolicy.RetryCondition.NO_RETRY_CONDITION, - new RetryPolicy.BackoffStrategy() { - @Override - public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest, - AmazonClientException exception, - int retriesAttempted) { - // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000) - logger.warn("EC2 API request failed, retry again. Reason was:", exception); - return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble())); - } - }, - 10, - false); + RetryPolicy.RetryCondition.NO_RETRY_CONDITION, + new RetryPolicy.BackoffStrategy() { + @Override + public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest, + AmazonClientException exception, + int retriesAttempted) { + // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000) + logger.warn("EC2 API request failed, retry again. Reason was:", exception); + return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble())); + } + }, + 10, + false); clientConfiguration.setRetryPolicy(retryPolicy); - AWSCredentialsProvider credentials; - - if (key == null && secret == null) { - credentials = new AWSCredentialsProviderChain( - new EnvironmentVariableCredentialsProvider(), - new SystemPropertiesCredentialsProvider(), - new InstanceProfileCredentialsProvider() - ); - } else { - credentials = new AWSCredentialsProviderChain( - new StaticCredentialsProvider(new BasicAWSCredentials(key, secret)) - ); - } - - this.client = new AmazonEC2Client(credentials, clientConfiguration); + return clientConfiguration; + } + protected static String findEndpoint(ESLogger logger, Settings settings) { + String endpoint = null; if (CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) { - final String endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings); + endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings); logger.debug("using explicit ec2 endpoint [{}]", endpoint); - client.setEndpoint(endpoint); - } else if (CLOUD_EC2.REGION_SETTING.exists(settings)) { + } else if (REGION_SETTING.exists(settings) || CLOUD_EC2.REGION_SETTING.exists(settings)) { final String region = CLOUD_EC2.REGION_SETTING.get(settings); - final String endpoint; - if (region.equals("us-east-1") || region.equals("us-east")) { - endpoint = "ec2.us-east-1.amazonaws.com"; - } else if (region.equals("us-west") || region.equals("us-west-1")) { - endpoint = "ec2.us-west-1.amazonaws.com"; - } else if (region.equals("us-west-2")) { - endpoint = "ec2.us-west-2.amazonaws.com"; - } else if (region.equals("ap-southeast") || region.equals("ap-southeast-1")) { - endpoint = "ec2.ap-southeast-1.amazonaws.com"; - } else if (region.equals("us-gov-west") || region.equals("us-gov-west-1")) { - endpoint = "ec2.us-gov-west-1.amazonaws.com"; - } else if (region.equals("ap-south-1")) { - endpoint = "ec2.ap-south-1.amazonaws.com"; - } else if (region.equals("ap-southeast-2")) { - endpoint = "ec2.ap-southeast-2.amazonaws.com"; - } else if (region.equals("ap-northeast") || region.equals("ap-northeast-1")) { - endpoint = "ec2.ap-northeast-1.amazonaws.com"; - } else if (region.equals("ap-northeast-2")) { - endpoint = "ec2.ap-northeast-2.amazonaws.com"; - } else if (region.equals("eu-west") || region.equals("eu-west-1")) { - endpoint = "ec2.eu-west-1.amazonaws.com"; - } else if (region.equals("eu-central") || region.equals("eu-central-1")) { - endpoint = "ec2.eu-central-1.amazonaws.com"; - } else if (region.equals("sa-east") || region.equals("sa-east-1")) { - endpoint = "ec2.sa-east-1.amazonaws.com"; - } else if (region.equals("cn-north") || region.equals("cn-north-1")) { - endpoint = "ec2.cn-north-1.amazonaws.com.cn"; - } else { - throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); + switch (region) { + case "us-east-1": + case "us-east": + endpoint = "ec2.us-east-1.amazonaws.com"; + break; + case "us-west": + case "us-west-1": + endpoint = "ec2.us-west-1.amazonaws.com"; + break; + case "us-west-2": + endpoint = "ec2.us-west-2.amazonaws.com"; + break; + case "ap-southeast": + case "ap-southeast-1": + endpoint = "ec2.ap-southeast-1.amazonaws.com"; + break; + case "ap-south-1": + endpoint = "ec2.ap-south-1.amazonaws.com"; + break; + case "us-gov-west": + case "us-gov-west-1": + endpoint = "ec2.us-gov-west-1.amazonaws.com"; + break; + case "ap-southeast-2": + endpoint = "ec2.ap-southeast-2.amazonaws.com"; + break; + case "ap-northeast": + case "ap-northeast-1": + endpoint = "ec2.ap-northeast-1.amazonaws.com"; + break; + case "ap-northeast-2": + endpoint = "ec2.ap-northeast-2.amazonaws.com"; + break; + case "eu-west": + case "eu-west-1": + endpoint = "ec2.eu-west-1.amazonaws.com"; + break; + case "eu-central": + case "eu-central-1": + endpoint = "ec2.eu-central-1.amazonaws.com"; + break; + case "sa-east": + case "sa-east-1": + endpoint = "ec2.sa-east-1.amazonaws.com"; + break; + case "cn-north": + case "cn-north-1": + endpoint = "ec2.cn-north-1.amazonaws.com.cn"; + break; + default: + throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]"); } logger.debug("using ec2 region [{}], with endpoint [{}]", region, endpoint); - client.setEndpoint(endpoint); } - - return this.client; + return endpoint; } @Override diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java new file mode 100644 index 00000000000..7ee82516926 --- /dev/null +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImplTests.java @@ -0,0 +1,169 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class AwsEc2ServiceImplTests extends ESTestCase { + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/19556") + public void testAWSCredentialsWithSystemProviders() { + AWSCredentialsProvider credentialsProvider = AwsEc2ServiceImpl.buildCredentials(logger, Settings.EMPTY); + + AWSCredentials credentials = credentialsProvider.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("DUMMY_ACCESS_KEY")); + assertThat(credentials.getAWSSecretKey(), is("DUMMY_SECRET_KEY")); + } + + public void testAWSCredentialsWithElasticsearchAwsSettings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret"); + } + + public void testAWSCredentialsWithElasticsearchEc2Settings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key") + .put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret"); + } + + public void testAWSCredentialsWithElasticsearchAwsAndEc2Settings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsEc2Service.SECRET_SETTING.getKey(), "aws_secret") + .put(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey(), "ec2_key") + .put(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey(), "ec2_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(settings, "ec2_key", "ec2_secret"); + } + + protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) { + AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, settings).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is(expectedKey)); + assertThat(credentials.getAWSSecretKey(), is(expectedSecret)); + } + + public void testAWSDefaultConfiguration() { + launchAWSConfigurationTest(Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null); + } + + public void testAWSConfigurationWithAwsSettings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http") + .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") + .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080) + .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") + .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") + .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType") + .build(); + launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password", + "AWS3SignerType"); + } + + public void testAWSConfigurationWithAwsAndEc2Settings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.PROTOCOL_SETTING.getKey(), "http") + .put(AwsEc2Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") + .put(AwsEc2Service.PROXY_PORT_SETTING.getKey(), 8080) + .put(AwsEc2Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") + .put(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") + .put(AwsEc2Service.SIGNER_SETTING.getKey(), "AWS3SignerType") + .put(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING.getKey(), "https") + .put(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING.getKey(), "ec2_proxy_host") + .put(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING.getKey(), 8081) + .put(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING.getKey(), "ec2_proxy_username") + .put(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.getKey(), "ec2_proxy_password") + .put(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING.getKey(), "NoOpSignerType") + .build(); + launchAWSConfigurationTest(settings, Protocol.HTTPS, "ec2_proxy_host", 8081, "ec2_proxy_username", "ec2_proxy_password", + "NoOpSignerType"); + } + + protected void launchAWSConfigurationTest(Settings settings, + Protocol expectedProtocol, + String expectedProxyHost, + int expectedProxyPort, + String expectedProxyUsername, + String expectedProxyPassword, + String expectedSigner) { + ClientConfiguration configuration = AwsEc2ServiceImpl.buildConfiguration(logger, settings); + + assertThat(configuration.getResponseMetadataCacheSize(), is(0)); + assertThat(configuration.getProtocol(), is(expectedProtocol)); + assertThat(configuration.getProxyHost(), is(expectedProxyHost)); + assertThat(configuration.getProxyPort(), is(expectedProxyPort)); + assertThat(configuration.getProxyUsername(), is(expectedProxyUsername)); + assertThat(configuration.getProxyPassword(), is(expectedProxyPassword)); + assertThat(configuration.getSignerOverride(), is(expectedSigner)); + } + + public void testDefaultEndpoint() { + String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, Settings.EMPTY); + assertThat(endpoint, nullValue()); + } + + public void testSpecificEndpoint() { + Settings settings = Settings.builder() + .put(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING.getKey(), "ec2.endpoint") + .build(); + String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings); + assertThat(endpoint, is("ec2.endpoint")); + } + + public void testRegionWithAwsSettings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1")) + .build(); + String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings); + assertThat(endpoint, is("ec2.eu-west-1.amazonaws.com")); + } + + public void testRegionWithAwsAndEc2Settings() { + Settings settings = Settings.builder() + .put(AwsEc2Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1")) + .put(AwsEc2Service.CLOUD_EC2.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1")) + .build(); + String endpoint = AwsEc2ServiceImpl.findEndpoint(logger, settings); + assertThat(endpoint, is("ec2.us-west-1.amazonaws.com")); + } + + public void testInvalidRegion() { + Settings settings = Settings.builder() + .put(AwsEc2Service.REGION_SETTING.getKey(), "does-not-exist") + .build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + AwsEc2ServiceImpl.findEndpoint(logger, settings); + }); + assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region")); + } +} diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java index 24ccf82a3d8..cc621dd5949 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.cloud.aws; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class DiscoveryEc2RestIT extends ESRestTestCase { +public class DiscoveryEc2RestIT extends ESClientYamlSuiteTestCase { public DiscoveryEc2RestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class DiscoveryEc2RestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGCERestIT.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGCERestIT.java index 891dd156aac..dae943f7bd2 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGCERestIT.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGCERestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.discovery.gce; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class DiscoveryGCERestIT extends ESRestTestCase { +public class DiscoveryGCERestIT extends ESClientYamlSuiteTestCase { public DiscoveryGCERestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class DiscoveryGCERestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentRestIT.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentRestIT.java index 2399f854c91..671b5b0ab95 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentRestIT.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.ingest.attachment; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class IngestAttachmentRestIT extends ESRestTestCase { +public class IngestAttachmentRestIT extends ESClientYamlSuiteTestCase { public IngestAttachmentRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class IngestAttachmentRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpRestIT.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpRestIT.java index 0e4d1ee4b2b..19cd7226d43 100644 --- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpRestIT.java +++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpRestIT.java @@ -22,14 +22,14 @@ package org.elasticsearch.ingest.geoip; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; import java.util.Collection; -public class IngestGeoIpRestIT extends ESRestTestCase { +public class IngestGeoIpRestIT extends ESClientYamlSuiteTestCase { public IngestGeoIpRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -37,7 +37,7 @@ public class IngestGeoIpRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java index 010f85f671a..0ca96698e3b 100644 --- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java +++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.ingest.useragent; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class UserAgentRestIT extends ESRestTestCase { +public class UserAgentRestIT extends ESClientYamlSuiteTestCase { public UserAgentRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,6 +35,6 @@ public class UserAgentRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java index 74573a79289..62be8332439 100644 --- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java +++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.plugin.example; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class JvmExampleRestIT extends ESRestTestCase { +public class JvmExampleRestIT extends ESClientYamlSuiteTestCase { public JvmExampleRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class JvmExampleRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java index 8039715c3d1..014c7073e71 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.script.javascript; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class LangJavaScriptRestIT extends ESRestTestCase { +public class LangJavaScriptRestIT extends ESClientYamlSuiteTestCase { public LangJavaScriptRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class LangJavaScriptRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java index ee0a707644f..cbf7547a580 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.script.python; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class LangPythonScriptRestIT extends ESRestTestCase { +public class LangPythonScriptRestIT extends ESClientYamlSuiteTestCase { public LangPythonScriptRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class LangPythonScriptRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java index 1eccb1a1445..0958225ae18 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperAttachmentsRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.mapper.attachments; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class MapperAttachmentsRestIT extends ESRestTestCase { +public class MapperAttachmentsRestIT extends ESClientYamlSuiteTestCase { public MapperAttachmentsRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java index bbe342c716c..37a4490d38c 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.mapper.murmur3; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class MapperMurmur3RestIT extends ESRestTestCase { +public class MapperMurmur3RestIT extends ESClientYamlSuiteTestCase { public MapperMurmur3RestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java index 84df085f221..f5ee35c5638 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.mapper.size; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class MapperSizeRestIT extends ESRestTestCase { +public class MapperSizeRestIT extends ESClientYamlSuiteTestCase { public MapperSizeRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java index 17ff0780a50..4e5dfb3efd5 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java @@ -27,15 +27,12 @@ import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.ListBlobItem; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.RepositoryException; @@ -163,13 +160,6 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS public void removeContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException { CloudBlobClient client = this.getSelectedClient(account, mode); CloudBlobContainer blobContainer = client.getContainerReference(container); - // TODO Should we set some timeout and retry options? - /* - BlobRequestOptions options = new BlobRequestOptions(); - options.setTimeoutIntervalInMs(1000); - options.setRetryPolicyFactory(new RetryNoRetry()); - blobContainer.deleteIfExists(options, null); - */ logger.trace("removing container [{}]", container); blobContainer.deleteIfExists(); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryRestIT.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryRestIT.java index ad58838bf5b..3c565d407ce 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryRestIT.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.repositories.azure; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class AzureRepositoryRestIT extends ESRestTestCase { +public class AzureRepositoryRestIT extends ESClientYamlSuiteTestCase { public AzureRepositoryRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class AzureRepositoryRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java index 18862d05aa0..b19b8623ace 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.repositories.gcs; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class GoogleCloudStorageRepositoryRestIT extends ESRestTestCase { +public class GoogleCloudStorageRepositoryRestIT extends ESClientYamlSuiteTestCase { public GoogleCloudStorageRepositoryRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryRestIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryRestIT.java index dea6e8b749f..147838df9ea 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryRestIT.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryRestIT.java @@ -22,11 +22,11 @@ import java.io.IOException; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; -public class HdfsRepositoryRestIT extends ESRestTestCase { +public class HdfsRepositoryRestIT extends ESClientYamlSuiteTestCase { public HdfsRepositoryRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -34,6 +34,6 @@ public class HdfsRepositoryRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index f713d8eadc5..23aa68e7f2d 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -50,7 +50,11 @@ dependencyLicenses { test { // this is needed for insecure plugins, remove if possible! - systemProperty 'tests.artifact', project.name + systemProperty 'tests.artifact', project.name + // this could be needed by AwsS3ServiceImplTests#testAWSCredentialsWithSystemProviders() + // As it's marked as Ignored for now, we can comment those + // systemProperty 'aws.accessKeyId', 'DUMMY_ACCESS_KEY' + // systemProperty 'aws.secretKey', 'DUMMY_SECRET_KEY' } thirdPartyAudit.excludes = [ diff --git a/plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1 deleted file mode 100644 index ef0c257e012..00000000000 --- a/plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b31526a230871fbe285fbcbe2813f9c0839ae9b0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 new file mode 100644 index 00000000000..58172660174 --- /dev/null +++ b/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 @@ -0,0 +1 @@ +e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 36d383d865c..e1bce876c27 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import java.util.HashMap; @@ -57,30 +58,33 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements } @Override - public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, + public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String key, String secret, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess) { - if (Strings.isNullOrEmpty(endpoint)) { - // We need to set the endpoint based on the region - if (region != null) { - endpoint = getEndpoint(region); - logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); - } else { - // No region has been set so we will use the default endpoint - endpoint = getDefaultEndpoint(); - } - } - - return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries, pathStyleAccess); - } - - private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries, - boolean useThrottleRetries, Boolean pathStyleAccess) { - Tuple clientDescriptor = new Tuple<>(endpoint, account); + String foundEndpoint = findEndpoint(logger, settings, endpoint, region); + Tuple clientDescriptor = new Tuple<>(foundEndpoint, key); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { return client; } + client = new AmazonS3Client( + buildCredentials(logger, key, secret), + buildConfiguration(logger, settings, protocol, maxRetries, foundEndpoint, useThrottleRetries)); + + if (pathStyleAccess != null) { + client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess)); + } + + if (!foundEndpoint.isEmpty()) { + client.setEndpoint(foundEndpoint); + } + + clients.put(clientDescriptor, client); + return client; + } + + public static ClientConfiguration buildConfiguration(ESLogger logger, Settings settings, Protocol protocol, Integer maxRetries, + String endpoint, boolean useThrottleRetries) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. @@ -113,43 +117,50 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint); } - AWSCredentialsProvider credentials; - - if (account == null && key == null) { - credentials = new AWSCredentialsProviderChain( - new EnvironmentVariableCredentialsProvider(), - new SystemPropertiesCredentialsProvider(), - new InstanceProfileCredentialsProvider() - ); - } else { - credentials = new AWSCredentialsProviderChain( - new StaticCredentialsProvider(new BasicAWSCredentials(account, key)) - ); - } - client = new AmazonS3Client(credentials, clientConfiguration); - - if (endpoint != null) { - client.setEndpoint(endpoint); - } - - if (pathStyleAccess != null) { - client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess)); - } - - clients.put(clientDescriptor, client); - return client; + return clientConfiguration; } - private String getDefaultEndpoint() { - String endpoint = null; - if (CLOUD_S3.ENDPOINT_SETTING.exists(settings)) { - endpoint = CLOUD_S3.ENDPOINT_SETTING.get(settings); - logger.debug("using explicit s3 endpoint [{}]", endpoint); - } else if (CLOUD_S3.REGION_SETTING.exists(settings)) { - String region = CLOUD_S3.REGION_SETTING.get(settings); - endpoint = getEndpoint(region); - logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + public static AWSCredentialsProvider buildCredentials(ESLogger logger, String key, String secret) { + AWSCredentialsProvider credentials; + + if (key.isEmpty() && secret.isEmpty()) { + logger.debug("Using either environment variables, system properties or instance profile credentials"); + credentials = new AWSCredentialsProviderChain( + new EnvironmentVariableCredentialsProvider(), + new SystemPropertiesCredentialsProvider(), + new InstanceProfileCredentialsProvider() + ); + } else { + logger.debug("Using basic key/secret credentials"); + credentials = new AWSCredentialsProviderChain( + new StaticCredentialsProvider(new BasicAWSCredentials(key, secret)) + ); } + + return credentials; + } + + protected static String findEndpoint(ESLogger logger, Settings settings, String endpoint, String region) { + if (Strings.isNullOrEmpty(endpoint)) { + logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region); + if (!region.isEmpty()) { + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } else { + // No region has been set so we will use the default endpoint + if (CLOUD_S3.ENDPOINT_SETTING.exists(settings)) { + endpoint = CLOUD_S3.ENDPOINT_SETTING.get(settings); + logger.debug("using explicit s3 endpoint [{}]", endpoint); + } else if (REGION_SETTING.exists(settings) || CLOUD_S3.REGION_SETTING.exists(settings)) { + region = CLOUD_S3.REGION_SETTING.get(settings); + endpoint = getEndpoint(region); + logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint); + } + } + } else { + logger.debug("using repository level endpoint [{}]", endpoint); + } + return endpoint; } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index cc91173d954..56d05b65711 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -270,11 +270,6 @@ public class S3Repository extends BlobStoreRepository { String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING); Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING); String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING); - // If no region is defined either in region, repositories.s3.region, cloud.aws.s3.region or cloud.aws.region - // we fallback to Default bucket - null - if (Strings.isEmpty(region)) { - region = null; - } boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING); ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java new file mode 100644 index 00000000000..788ea8b60ed --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java @@ -0,0 +1,313 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.repositories.s3.S3Repository; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +public class AwsS3ServiceImplTests extends ESTestCase { + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/19556") + public void testAWSCredentialsWithSystemProviders() { + AWSCredentialsProvider credentialsProvider = InternalAwsS3Service.buildCredentials(logger, "", ""); + + AWSCredentials credentials = credentialsProvider.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("DUMMY_ACCESS_KEY")); + assertThat(credentials.getAWSSecretKey(), is("DUMMY_SECRET_KEY")); + } + + public void testAWSCredentialsWithElasticsearchAwsSettings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret"); + } + + public void testAWSCredentialsWithElasticsearchS3Settings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key") + .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret"); + } + + public void testAWSCredentialsWithElasticsearchAwsAndS3Settings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret") + .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key") + .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret"); + } + + public void testAWSCredentialsWithElasticsearchRepositoriesSettings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key") + .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); + } + + public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret") + .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key") + .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); + } + + public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret") + .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key") + .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret") + .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key") + .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret"); + } + + public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettings() { + Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null); + Settings settings = Settings.builder() + .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key") + .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret"); + } + + public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettings() { + Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret") + .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key") + .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret"); + } + + public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettings() { + Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key") + .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret") + .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key") + .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret") + .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key") + .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret") + .build(); + launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret"); + } + + protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings, + String expectedKey, String expectedSecret) { + String key = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING); + String secret = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING); + + AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, key, secret).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is(expectedKey)); + assertThat(credentials.getAWSSecretKey(), is(expectedSecret)); + } + + public void testAWSDefaultConfiguration() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + launchAWSConfigurationTest(Settings.EMPTY, repositorySettings, Protocol.HTTPS, null, -1, null, null, null, 3, false); + } + + public void testAWSConfigurationWithAwsSettings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http") + .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") + .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080) + .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") + .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") + .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") + .build(); + launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", + "aws_proxy_password", "AWS3SignerType", 3, false); + } + + public void testAWSConfigurationWithAwsAndS3Settings() { + Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null); + Settings settings = Settings.builder() + .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http") + .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host") + .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 8080) + .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username") + .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password") + .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType") + .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https") + .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host") + .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081) + .put(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3_proxy_username") + .put(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3_proxy_password") + .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType") + .build(); + launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username", + "s3_proxy_password", "NoOpSignerType", 3, false); + } + + protected void launchAWSConfigurationTest(Settings settings, + Settings singleRepositorySettings, + Protocol expectedProtocol, + String expectedProxyHost, + int expectedProxyPort, + String expectedProxyUsername, + String expectedProxyPassword, + String expectedSigner, + Integer expectedMaxRetries, + boolean expectedUseThrottleRetries) { + Protocol protocol = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING); + Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING); + Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING); + + ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, settings, protocol, maxRetries, null, + useThrottleRetries); + + assertThat(configuration.getResponseMetadataCacheSize(), is(0)); + assertThat(configuration.getProtocol(), is(expectedProtocol)); + assertThat(configuration.getProxyHost(), is(expectedProxyHost)); + assertThat(configuration.getProxyPort(), is(expectedProxyPort)); + assertThat(configuration.getProxyUsername(), is(expectedProxyUsername)); + assertThat(configuration.getProxyPassword(), is(expectedProxyPassword)); + assertThat(configuration.getSignerOverride(), is(expectedSigner)); + assertThat(configuration.getMaxErrorRetry(), is(expectedMaxRetries)); + assertThat(configuration.useThrottledRetries(), is(expectedUseThrottleRetries)); + } + + private static Settings generateRepositorySettings(String key, String secret, String region, String endpoint, Integer maxRetries) { + Settings.Builder builder = Settings.builder(); + if (region != null) { + builder.put(S3Repository.Repository.REGION_SETTING.getKey(), region); + } + if (endpoint != null) { + builder.put(S3Repository.Repository.ENDPOINT_SETTING.getKey(), endpoint); + } + if (key != null) { + builder.put(S3Repository.Repository.KEY_SETTING.getKey(), key); + } + if (secret != null) { + builder.put(S3Repository.Repository.SECRET_SETTING.getKey(), secret); + } + if (maxRetries != null) { + builder.put(S3Repository.Repository.MAX_RETRIES_SETTING.getKey(), maxRetries); + } + return builder.build(); + } + + public void testDefaultEndpoint() { + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, ""); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), Settings.EMPTY, + "s3.eu-central-1.amazonaws.com"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + Settings.EMPTY, "repository.endpoint"); + } + + public void testSpecificEndpoint() { + Settings settings = Settings.builder() + .put(InternalAwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "ec2.endpoint") + .build(); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + "ec2.endpoint"); + // Endpoint has precedence on region. Whatever region we set, we won't use it + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, + "ec2.endpoint"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + settings, "repository.endpoint"); + } + + public void testRegionWithAwsSettings() { + Settings settings = Settings.builder() + .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1")) + .build(); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + "s3-eu-west-1.amazonaws.com"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, + "s3.eu-central-1.amazonaws.com"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + settings, "repository.endpoint"); + } + + public void testRegionWithAwsAndS3Settings() { + Settings settings = Settings.builder() + .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1")) + .put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1")) + .build(); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, + "s3-us-west-1.amazonaws.com"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, + "s3.eu-central-1.amazonaws.com"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + settings, "repository.endpoint"); + } + + public void testInvalidRegion() { + Settings settings = Settings.builder() + .put(InternalAwsS3Service.REGION_SETTING.getKey(), "does-not-exist") + .build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null); + }); + assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region")); + + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings, + "s3.eu-central-1.amazonaws.com"); + launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null), + settings, "repository.endpoint"); + } + + protected void launchAWSEndpointTest(Settings singleRepositorySettings, Settings settings, + String expectedEndpoint) { + String region = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING); + String endpoint = S3Repository.getValue(singleRepositorySettings, settings, + S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING); + + String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, settings, endpoint, region); + assertThat(foundEndpoint, is(expectedEndpoint)); + } + +} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java index d8e436b50bb..5dada2a2637 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.repositories.s3; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class RepositoryS3RestIT extends ESRestTestCase { +public class RepositoryS3RestIT extends ESClientYamlSuiteTestCase { public RepositoryS3RestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class RepositoryS3RestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SMBStoreRestIT.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SMBStoreRestIT.java index af1b0372995..99617b94644 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SMBStoreRestIT.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SMBStoreRestIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.index.store; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class SMBStoreRestIT extends ESRestTestCase { +public class SMBStoreRestIT extends ESClientYamlSuiteTestCase { public SMBStoreRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class SMBStoreRestIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java index 1f3ad15d1bf..e23ca840025 100644 --- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java +++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java @@ -22,14 +22,14 @@ package org.elasticsearch.backwards; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import org.apache.lucene.util.TimeUnits; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell -public class MultiNodeBackwardsIT extends ESRestTestCase { +public class MultiNodeBackwardsIT extends ESClientYamlSuiteTestCase { public MultiNodeBackwardsIT(RestTestCandidate testCandidate) { super(testCandidate); diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java index 5f54d14eca2..96c1139c790 100644 --- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java +++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.transport.MockTcpTransportPlugin; +import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.junit.After; import org.junit.AfterClass; @@ -49,6 +50,8 @@ import java.util.Locale; import java.util.concurrent.atomic.AtomicInteger; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; import static org.hamcrest.Matchers.notNullValue; /** @@ -82,13 +85,22 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase { .put("client.transport.ignore_cluster_name", true) .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); final Collection> plugins; - if (random().nextBoolean()) { - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME); - plugins = Collections.singleton(MockTcpTransportPlugin.class); - } else { - plugins = Collections.emptyList(); + switch (randomIntBetween(0, 2)) { + case 0: + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME); + plugins = Collections.singleton(MockTcpTransportPlugin.class); + break; + case 1: + plugins = Collections.emptyList(); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME); + break; + case 2: + plugins = Collections.emptyList(); + break; + default: + throw new AssertionError(); } - TransportClient client = new PreBuiltTransportClient(builder.build(), plugins).addTransportAddresses(transportAddresses); + TransportClient client = new PreBuiltTransportClient(builder.build(), plugins).addTransportAddresses(transportAddresses); logger.info("--> Elasticsearch Java TransportClient started"); diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle index 7fa3205537a..038115c2d24 100644 --- a/qa/smoke-test-http/build.gradle +++ b/qa/smoke-test-http/build.gradle @@ -21,4 +21,5 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: ':modules:transport-netty3', configuration: 'runtime') // for http + testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for http } \ No newline at end of file diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java index caaa328b1ab..1249f1deadf 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java @@ -51,6 +51,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -218,20 +219,17 @@ public class ContextAndHeaderTransportIT extends HttpSmokeTestCase { assertRequestsContainHeader(MultiTermVectorsRequest.class); } - public void testThatRelevantHttpHeadersBecomeRequestHeaders() throws Exception { + public void testThatRelevantHttpHeadersBecomeRequestHeaders() throws IOException { final String IRRELEVANT_HEADER = "SomeIrrelevantHeader"; - - try (Response response = getRestClient().performRequest( - "GET", "/" + queryIndex + "/_search", - new BasicHeader(CUSTOM_HEADER, randomHeaderValue), new BasicHeader(IRRELEVANT_HEADER, randomHeaderValue))) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); - List searchRequests = getRequests(SearchRequest.class); - assertThat(searchRequests, hasSize(greaterThan(0))); - for (RequestAndHeaders requestAndHeaders : searchRequests) { - assertThat(requestAndHeaders.headers.containsKey(CUSTOM_HEADER), is(true)); - // was not specified, thus is not included - assertThat(requestAndHeaders.headers.containsKey(IRRELEVANT_HEADER), is(false)); - } + Response response = getRestClient().performRequest("GET", "/" + queryIndex + "/_search", + new BasicHeader(CUSTOM_HEADER, randomHeaderValue), new BasicHeader(IRRELEVANT_HEADER, randomHeaderValue)); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + List searchRequests = getRequests(SearchRequest.class); + assertThat(searchRequests, hasSize(greaterThan(0))); + for (RequestAndHeaders requestAndHeaders : searchRequests) { + assertThat(requestAndHeaders.headers.containsKey(CUSTOM_HEADER), is(true)); + // was not specified, thus is not included + assertThat(requestAndHeaders.headers.containsKey(IRRELEVANT_HEADER), is(false)); } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsNotSetIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsNotSetIT.java index 7cc84354f6f..bdda44c1b71 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsNotSetIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsNotSetIT.java @@ -23,27 +23,26 @@ import org.apache.http.message.BasicHeader; import org.elasticsearch.client.Response; import org.elasticsearch.test.ESIntegTestCase; +import java.io.IOException; + import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; public class CorsNotSetIT extends HttpSmokeTestCase { - - public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception { + public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws IOException { String corsValue = "http://localhost:9200"; - try (Response response = getRestClient().performRequest("GET", "/", - new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) { - assertThat(response.getStatusLine().getStatusCode(), is(200)); - assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); - assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue()); - } + Response response = getRestClient().performRequest("GET", "/", + new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue)); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); + assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue()); } - public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception { - try (Response response = getRestClient().performRequest("GET", "/")) { - assertThat(response.getStatusLine().getStatusCode(), is(200)); - assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); - assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue()); - } + public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws IOException { + Response response = getRestClient().performRequest("GET", "/"); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); + assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue()); } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsRegexIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsRegexIT.java index a7d3f4156df..ca5cc1d0762 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsRegexIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/CorsRegexIT.java @@ -26,6 +26,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.jboss.netty.handler.codec.http.HttpHeaders; + +import java.io.IOException; import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS; import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS; @@ -57,21 +60,20 @@ public class CorsRegexIT extends HttpSmokeTestCase { return true; } - public void testThatRegularExpressionWorksOnMatch() throws Exception { + public void testThatRegularExpressionWorksOnMatch() throws IOException { String corsValue = "http://localhost:9200"; - try (Response response = getRestClient().performRequest("GET", "/", - new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) { - assertResponseWithOriginheader(response, corsValue); - } + Response response = getRestClient().performRequest("GET", "/", + new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue)); + assertResponseWithOriginheader(response, corsValue); + corsValue = "https://localhost:9200"; - try (Response response = getRestClient().performRequest("GET", "/", - new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) { - assertResponseWithOriginheader(response, corsValue); - assertThat(response.getHeader("Access-Control-Allow-Credentials"), is("true")); - } + response = getRestClient().performRequest("GET", "/", + new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue)); + assertResponseWithOriginheader(response, corsValue); + assertThat(response.getHeader("Access-Control-Allow-Credentials"), is("true")); } - public void testThatRegularExpressionReturnsForbiddenOnNonMatch() throws Exception { + public void testThatRegularExpressionReturnsForbiddenOnNonMatch() throws IOException { try { getRestClient().performRequest("GET", "/", new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", "http://evil-host:9200")); @@ -84,31 +86,28 @@ public class CorsRegexIT extends HttpSmokeTestCase { } } - public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception { - try (Response response = getRestClient().performRequest("GET", "/", new BasicHeader("User-Agent", "Mozilla Bar"))) { - assertThat(response.getStatusLine().getStatusCode(), is(200)); - assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); - } + public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws IOException { + Response response = getRestClient().performRequest("GET", "/", new BasicHeader("User-Agent", "Mozilla Bar")); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); } - public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception { - try (Response response = getRestClient().performRequest("GET", "/")) { - assertThat(response.getStatusLine().getStatusCode(), is(200)); - assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); - } + public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws IOException { + Response response = getRestClient().performRequest("GET", "/"); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue()); } - public void testThatPreFlightRequestWorksOnMatch() throws Exception { + public void testThatPreFlightRequestWorksOnMatch() throws IOException { String corsValue = "http://localhost:9200"; - try (Response response = getRestClient().performRequest("OPTIONS", "/", + Response response = getRestClient().performRequest("OPTIONS", "/", new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue), - new BasicHeader("Access-Control-Request-Method", "GET"));) { - assertResponseWithOriginheader(response, corsValue); - assertNotNull(response.getHeader("Access-Control-Allow-Methods")); - } + new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET")); + assertResponseWithOriginheader(response, corsValue); + assertNotNull(response.getHeader("Access-Control-Allow-Methods")); } - public void testThatPreFlightRequestReturnsNullOnNonMatch() throws Exception { + public void testThatPreFlightRequestReturnsNullOnNonMatch() throws IOException { try { getRestClient().performRequest("OPTIONS", "/", new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", "http://evil-host:9200"), diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java index f3b5d214fa4..778daa7714e 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java @@ -20,16 +20,15 @@ package org.elasticsearch.http; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.plugins.Plugin; - import org.hamcrest.Matcher; import java.io.IOException; @@ -99,35 +98,31 @@ public class DeprecationHttpIT extends HttpSmokeTestCase { final String commaSeparatedIndices = Stream.of(indices).collect(Collectors.joining(",")); - final String body = - "{\"query\":{\"bool\":{\"filter\":[{\"" + TestDeprecatedQueryBuilder.NAME + "\":{}}]}}}"; + final String body = "{\"query\":{\"bool\":{\"filter\":[{\"" + TestDeprecatedQueryBuilder.NAME + "\":{}}]}}}"; // trigger all index deprecations - try (Response response = getRestClient().performRequest("GET", - "/" + commaSeparatedIndices + "/_search", - Collections.emptyMap(), - new StringEntity(body, RestClient.JSON_CONTENT_TYPE))) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + Response response = getRestClient().performRequest("GET", "/" + commaSeparatedIndices + "/_search", + Collections.emptyMap(), new StringEntity(body, ContentType.APPLICATION_JSON)); + assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); - final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); - final List> headerMatchers = new ArrayList<>(indices.length); + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + final List> headerMatchers = new ArrayList<>(indices.length); - for (String index : indices) { - headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] index", (Object)index))); - } + for (String index : indices) { + headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] index", (Object)index))); + } - assertThat(deprecatedWarnings, hasSize(headerMatchers.size())); - for (Matcher headerMatcher : headerMatchers) { - assertThat(deprecatedWarnings, hasItem(headerMatcher)); - } + assertThat(deprecatedWarnings, hasSize(headerMatchers.size())); + for (Matcher headerMatcher : headerMatchers) { + assertThat(deprecatedWarnings, hasItem(headerMatcher)); } } - public void testDeprecationWarningsAppearInHeaders() throws IOException { + public void testDeprecationWarningsAppearInHeaders() throws Exception { doTestDeprecationWarningsAppearInHeaders(); } - public void testDeprecationHeadersDoNotGetStuck() throws IOException { + public void testDeprecationHeadersDoNotGetStuck() throws Exception { doTestDeprecationWarningsAppearInHeaders(); doTestDeprecationWarningsAppearInHeaders(); if (rarely()) { @@ -159,29 +154,26 @@ public class DeprecationHttpIT extends HttpSmokeTestCase { Collections.shuffle(settings, random()); // trigger all deprecations - try (Response response = getRestClient().performRequest("GET", - "/_test_cluster/deprecated_settings", - Collections.emptyMap(), - buildSettingsRequest(settings, useDeprecatedField))) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + Response response = getRestClient().performRequest("GET", "/_test_cluster/deprecated_settings", + Collections.emptyMap(), buildSettingsRequest(settings, useDeprecatedField)); + assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); - final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); - final List> headerMatchers = new ArrayList<>(4); + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + final List> headerMatchers = new ArrayList<>(4); - headerMatchers.add(equalTo(TestDeprecationHeaderRestAction.DEPRECATED_ENDPOINT)); - if (useDeprecatedField) { - headerMatchers.add(equalTo(TestDeprecationHeaderRestAction.DEPRECATED_USAGE)); - } - for (Setting setting : settings) { - if (setting.isDeprecated()) { - headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] setting was deprecated", (Object)setting.getKey()))); - } + headerMatchers.add(equalTo(TestDeprecationHeaderRestAction.DEPRECATED_ENDPOINT)); + if (useDeprecatedField) { + headerMatchers.add(equalTo(TestDeprecationHeaderRestAction.DEPRECATED_USAGE)); + } + for (Setting setting : settings) { + if (setting.isDeprecated()) { + headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] setting was deprecated", (Object)setting.getKey()))); } + } - assertThat(deprecatedWarnings, hasSize(headerMatchers.size())); - for (Matcher headerMatcher : headerMatchers) { - assertThat(deprecatedWarnings, hasItem(headerMatcher)); - } + assertThat(deprecatedWarnings, hasSize(headerMatchers.size())); + for (Matcher headerMatcher : headerMatchers) { + assertThat(deprecatedWarnings, hasItem(headerMatcher)); } } @@ -208,7 +200,7 @@ public class DeprecationHttpIT extends HttpSmokeTestCase { builder.endArray().endObject(); - return new StringEntity(builder.string(), RestClient.JSON_CONTENT_TYPE); + return new StringEntity(builder.string(), ContentType.APPLICATION_JSON); } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java index feca7cd1d5f..0fd4d6e6edd 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java @@ -19,15 +19,16 @@ package org.elasticsearch.http; +import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import java.io.IOException; import java.util.Collections; import static org.hamcrest.Matchers.is; @@ -52,14 +53,14 @@ public class DetailedErrorsDisabledIT extends HttpSmokeTestCase { return true; } - public void testThatErrorTraceParamReturns400() throws Exception { + public void testThatErrorTraceParamReturns400() throws IOException { try { getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true")); fail("request should have failed"); } catch(ResponseException e) { Response response = e.getResponse(); assertThat(response.getHeader("Content-Type"), is("application/json")); - assertThat(e.getResponseBody(), is("{\"error\":\"error traces in responses are disabled.\"}")); + assertThat(EntityUtils.toString(e.getResponse().getEntity()), is("{\"error\":\"error traces in responses are disabled.\"}")); assertThat(response.getStatusLine().getStatusCode(), is(400)); } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java index daabb1bc70d..cd3cf1a38d8 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java @@ -19,14 +19,12 @@ package org.elasticsearch.http; +import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.Scope; +import java.io.IOException; import java.util.Collections; import static org.hamcrest.Matchers.containsString; @@ -37,14 +35,15 @@ import static org.hamcrest.Matchers.not; */ public class DetailedErrorsEnabledIT extends HttpSmokeTestCase { - public void testThatErrorTraceWorksByDefault() throws Exception { + public void testThatErrorTraceWorksByDefault() throws IOException { try { getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true")); fail("request should have failed"); } catch(ResponseException e) { Response response = e.getResponse(); assertThat(response.getHeader("Content-Type"), containsString("application/json")); - assertThat(e.getResponseBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " + + assertThat(EntityUtils.toString(response.getEntity()), + containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " + "nested: ActionRequestValidationException[Validation Failed: 1:")); } @@ -54,7 +53,8 @@ public class DetailedErrorsEnabledIT extends HttpSmokeTestCase { } catch(ResponseException e) { Response response = e.getResponse(); assertThat(response.getHeader("Content-Type"), containsString("application/json")); - assertThat(e.getResponseBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " + assertThat(EntityUtils.toString(response.getEntity()), + not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " + "nested: ActionRequestValidationException[Validation Failed: 1:"))); } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java index f08bb2b4a9e..6e548dd2ca6 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java @@ -18,14 +18,10 @@ */ package org.elasticsearch.http; -import org.apache.http.Header; -import org.apache.http.HttpException; import org.apache.http.HttpHeaders; -import org.apache.http.HttpResponseInterceptor; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicHeader; -import org.apache.http.protocol.HttpContext; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.test.ESIntegTestCase; @@ -41,108 +37,33 @@ public class HttpCompressionIT extends ESIntegTestCase { " \"first name\": \"Steve\",\n" + " \"last name\": \"Jobs\"\n" + " }\n" + - "}", RestClient.JSON_CONTENT_TYPE); + "}", ContentType.APPLICATION_JSON); @Override protected boolean ignoreExternalCluster() { return false; } - public void testCompressesResponseIfRequested() throws Exception { + public void testCompressesResponseIfRequested() throws IOException { ensureGreen(); - // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it - ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor(); - try (RestClient client = createRestClient(new ContentEncodingHeaderExtractorConfigCallback(headerExtractor))) { - try (Response response = client.performRequest("GET", "/", new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING))) { - assertEquals(200, response.getStatusLine().getStatusCode()); - assertTrue(headerExtractor.hasContentEncodingHeader()); - assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue()); - } + try (RestClient client = getRestClient()) { + Response response = client.performRequest("GET", "/", new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING)); + assertEquals(200, response.getStatusLine().getStatusCode()); + assertEquals(GZIP_ENCODING, response.getHeader(HttpHeaders.CONTENT_ENCODING)); } } - public void testUncompressedResponseByDefault() throws Exception { + public void testUncompressedResponseByDefault() throws IOException { ensureGreen(); - ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor(); - try (RestClient client = createRestClient(new NoContentCompressionConfigCallback(headerExtractor))) { - try (Response response = client.performRequest("GET", "/")) { - assertEquals(200, response.getStatusLine().getStatusCode()); - assertFalse(headerExtractor.hasContentEncodingHeader()); - } + try (RestClient client = getRestClient()) { + Response response = client.performRequest("GET", "/"); + assertEquals(200, response.getStatusLine().getStatusCode()); + assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING)); + + response = client.performRequest("POST", "/company/employees/1", Collections.emptyMap(), SAMPLE_DOCUMENT); + assertEquals(201, response.getStatusLine().getStatusCode()); + assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING)); } } - public void testCanInterpretUncompressedRequest() throws Exception { - ensureGreen(); - ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor(); - // this disable content compression in both directions (request and response) - try (RestClient client = createRestClient(new NoContentCompressionConfigCallback(headerExtractor))) { - try (Response response = client.performRequest("POST", "/company/employees/1", - Collections.emptyMap(), SAMPLE_DOCUMENT)) { - assertEquals(201, response.getStatusLine().getStatusCode()); - assertFalse(headerExtractor.hasContentEncodingHeader()); - } - } - } - - public void testCanInterpretCompressedRequest() throws Exception { - ensureGreen(); - ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor(); - // we don't call #disableContentCompression() hence the client will send the content compressed - try (RestClient client = createRestClient(new ContentEncodingHeaderExtractorConfigCallback(headerExtractor))) { - try (Response response = client.performRequest("POST", "/company/employees/2", - Collections.emptyMap(), SAMPLE_DOCUMENT)) { - assertEquals(201, response.getStatusLine().getStatusCode()); - assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue()); - } - } - } - - private static class ContentEncodingHeaderExtractor implements HttpResponseInterceptor { - private Header contentEncodingHeader; - - @Override - public void process(org.apache.http.HttpResponse response, HttpContext context) throws HttpException, IOException { - final Header[] headers = response.getHeaders(HttpHeaders.CONTENT_ENCODING); - if (headers.length == 1) { - this.contentEncodingHeader = headers[0]; - } else if (headers.length > 1) { - throw new AssertionError("Expected none or one content encoding header but got " + headers.length + " headers."); - } - } - - public boolean hasContentEncodingHeader() { - return contentEncodingHeader != null; - } - - public Header getContentEncodingHeader() { - return contentEncodingHeader; - } - } - - private static class NoContentCompressionConfigCallback extends ContentEncodingHeaderExtractorConfigCallback { - NoContentCompressionConfigCallback(ContentEncodingHeaderExtractor contentEncodingHeaderExtractor) { - super(contentEncodingHeaderExtractor); - } - - @Override - public void customizeHttpClient(HttpClientBuilder httpClientBuilder) { - super.customizeHttpClient(httpClientBuilder); - httpClientBuilder.disableContentCompression(); - } - } - - private static class ContentEncodingHeaderExtractorConfigCallback implements RestClient.HttpClientConfigCallback { - - private final ContentEncodingHeaderExtractor contentEncodingHeaderExtractor; - - ContentEncodingHeaderExtractorConfigCallback(ContentEncodingHeaderExtractor contentEncodingHeaderExtractor) { - this.contentEncodingHeaderExtractor = contentEncodingHeaderExtractor; - } - - @Override - public void customizeHttpClient(HttpClientBuilder httpClientBuilder) { - httpClientBuilder.addInterceptorFirst(contentEncodingHeaderExtractor); - } - } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java index 636d652feb1..03a7ba68a71 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java @@ -25,6 +25,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.transport.Netty3Plugin; +import org.elasticsearch.transport.Netty4Plugin; +import org.junit.BeforeClass; import java.util.Collection; import java.util.Collections; @@ -32,33 +34,56 @@ import java.util.List; public abstract class HttpSmokeTestCase extends ESIntegTestCase { + private static String nodeTransportTypeKey; + private static String nodeHttpTypeKey; + private static String clientTypeKey; + + @SuppressWarnings("unchecked") + @BeforeClass + public static void setUpTransport() { + nodeTransportTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class)); + nodeHttpTypeKey = getTypeKey(randomFrom(Netty3Plugin.class, Netty4Plugin.class)); + clientTypeKey = getTypeKey(randomFrom(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class)); + } + + private static String getTypeKey(Class clazz) { + if (clazz.equals(MockTcpTransportPlugin.class)) { + return MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME; + } else if (clazz.equals(Netty3Plugin.class)) { + return Netty3Plugin.NETTY_TRANSPORT_NAME; + } else { + assert clazz.equals(Netty4Plugin.class); + return Netty4Plugin.NETTY_TRANSPORT_NAME; + } + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("netty.assert.buglevel", false) - .put(NetworkModule.TRANSPORT_TYPE_KEY, randomFrom(Netty3Plugin.NETTY_TRANSPORT_NAME, - MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)) - .put(NetworkModule.HTTP_ENABLED.getKey(), true).build(); + .put(super.nodeSettings(nodeOrdinal)) + .put("netty.assert.buglevel", false) + .put(NetworkModule.TRANSPORT_TYPE_KEY, nodeTransportTypeKey) + .put(NetworkModule.HTTP_TYPE_KEY, nodeHttpTypeKey) + .put(NetworkModule.HTTP_ENABLED.getKey(), true).build(); } @Override protected Collection> nodePlugins() { - return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, BogusPlugin.class); + return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class, BogusPlugin.class); } @Override protected Collection> transportClientPlugins() { - return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, BogusPlugin.class); + return pluginList(MockTcpTransportPlugin.class, Netty3Plugin.class, Netty4Plugin.class, BogusPlugin.class); } @Override protected Settings transportClientSettings() { return Settings.builder() - .put(super.transportClientSettings()) - .put("netty.assert.buglevel", false) - .put(NetworkModule.TRANSPORT_TYPE_KEY, randomFrom(Netty3Plugin.NETTY_TRANSPORT_NAME, - MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)).build(); + .put(super.transportClientSettings()) + .put("netty.assert.buglevel", false) + .put(NetworkModule.TRANSPORT_TYPE_KEY, clientTypeKey) + .build(); } @Override @@ -66,15 +91,18 @@ public abstract class HttpSmokeTestCase extends ESIntegTestCase { return true; } - public static final class BogusPlugin extends Plugin { - // see Netty3Plugin.... this runs without the permission from the netty3 module so it will fail since reindex can't set the property + + // this runs without the permission from the netty modules so it will fail since reindex can't set the property // to make it still work we disable that check but need to register the setting first - private static final Setting ASSERT_NETTY_BUGLEVEL = Setting.boolSetting("netty.assert.buglevel", true, - Setting.Property.NodeScope); + private static final Setting ASSERT_NETTY_BUGLEVEL = + Setting.boolSetting("netty.assert.buglevel", true, Setting.Property.NodeScope); + @Override public List> getSettings() { return Collections.singletonList(ASSERT_NETTY_BUGLEVEL); } + } + } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ResponseHeaderPluginIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ResponseHeaderPluginIT.java index 482edc36702..1e8b25b288d 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ResponseHeaderPluginIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ResponseHeaderPluginIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import java.util.ArrayList; +import java.io.IOException; import java.util.Collection; import static org.hamcrest.Matchers.equalTo; @@ -56,7 +57,7 @@ public class ResponseHeaderPluginIT extends HttpSmokeTestCase { return plugins; } - public void testThatSettingHeadersWorks() throws Exception { + public void testThatSettingHeadersWorks() throws IOException { ensureGreen(); try { getRestClient().performRequest("GET", "/_protected"); @@ -67,9 +68,8 @@ public class ResponseHeaderPluginIT extends HttpSmokeTestCase { assertThat(response.getHeader("Secret"), equalTo("required")); } - try (Response authResponse = getRestClient().performRequest("GET", "/_protected", new BasicHeader("Secret", "password"))) { - assertThat(authResponse.getStatusLine().getStatusCode(), equalTo(200)); - assertThat(authResponse.getHeader("Secret"), equalTo("granted")); - } + Response authResponse = getRestClient().performRequest("GET", "/_protected", new BasicHeader("Secret", "password")); + assertThat(authResponse.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(authResponse.getHeader("Secret"), equalTo("granted")); } } diff --git a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/IngestDisabledIT.java b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/IngestDisabledIT.java index e162807baca..d1cb62cc54b 100644 --- a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/IngestDisabledIT.java +++ b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/IngestDisabledIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class IngestDisabledIT extends ESRestTestCase { +public class IngestDisabledIT extends ESClientYamlSuiteTestCase { public IngestDisabledIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class IngestDisabledIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java index b9873214aa2..1ed23e7add5 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class IngestWithDependenciesIT extends ESRestTestCase { +public class IngestWithDependenciesIT extends ESClientYamlSuiteTestCase { public IngestWithDependenciesIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class IngestWithDependenciesIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiIT.java b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiIT.java index 75c4633e632..78d7acd8c24 100644 --- a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiIT.java +++ b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class SmokeTestMultiIT extends ESRestTestCase { +public class SmokeTestMultiIT extends ESClientYamlSuiteTestCase { public SmokeTestMultiIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class SmokeTestMultiIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java index 6e0243b2a04..61903e62993 100644 --- a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java +++ b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsIT.java @@ -21,13 +21,13 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class SmokeTestPluginsIT extends ESRestTestCase { +public class SmokeTestPluginsIT extends ESClientYamlSuiteTestCase { public SmokeTestPluginsIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); @@ -35,7 +35,7 @@ public class SmokeTestPluginsIT extends ESRestTestCase { @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java index 7857259a60d..b7061432325 100644 --- a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java +++ b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java @@ -21,19 +21,19 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class SmokeTestReindexWithPainlessIT extends ESRestTestCase { +public class SmokeTestReindexWithPainlessIT extends ESClientYamlSuiteTestCase { public SmokeTestReindexWithPainlessIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index 86523a734ad..303032299b2 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -101,7 +101,7 @@ dependencies { // Collect all the plugins for (Project subproj : project.rootProject.subprojects) { if (subproj.path.startsWith(':plugins:')) { - test project("${subproj.path}") + test project(path: "${subproj.path}", configuration: 'zip') } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml index 7f724831bd8..5f0646139b5 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.allocation/10_basic.yaml @@ -34,7 +34,7 @@ (\d+ \s+)? #no value from client nodes [-\w.]+ \s+ \d+(\.\d+){3} \s+ - \w.* + [-\w]+ \n )+ $/ @@ -61,7 +61,7 @@ (\d+ \s+) #always should return value since we filter out non data nodes by default [-\w.]+ \s+ \d+(\.\d+){3} \s+ - \w.* + [-\w]+ \n )+ ( @@ -90,7 +90,7 @@ (\d+ \s+)? #no value from client nodes [-\w.]+ \s+ \d+(\.\d+){3} \s+ - \w.* + [-\w]+ \n ) $/ @@ -124,7 +124,7 @@ (\d+ \s+)? #no value from client nodes [-\w.]+ \s+ \d+(\.\d+){3} \s+ - \w.* + [-\w]+ \n )+ ( @@ -164,7 +164,7 @@ (\d+ \s+) #always should return value since we filter out non data nodes by default [-\w.]+ \s+ \d+(\.\d+){3} \s+ - \w.* + [-\w]+ \n )+ $/ @@ -181,7 +181,7 @@ $body: | /^ ( \d* \s+ - \w.* + [-\w]+ \n )+ $/ @@ -199,7 +199,7 @@ \n ( \s+\d* \s+ - \w.* + [-\w]+ \n )+ $/ @@ -224,7 +224,7 @@ (\d+ \s+) #always should return value since we filter out non data nodes by default [-\w.]+ \s+ \d+(\.\d+){3} \s+ - \w.* + [-\w]+ \n )+ $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml index 3c6ad7a7051..cabfbfeb8b7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yaml @@ -45,6 +45,10 @@ - match: { shards_acknowledged: false } + - do: + cluster.health: + wait_for_events: urgent + --- "Create index with wait_for_active_shards set to all": diff --git a/settings.gradle b/settings.gradle index 36fa9a0907a..4475131a8dc 100644 --- a/settings.gradle +++ b/settings.gradle @@ -9,6 +9,7 @@ List projects = [ 'client:sniffer', 'client:transport', 'client:test', + 'client:benchmark', 'benchmarks', 'distribution:integ-test-zip', 'distribution:zip', @@ -26,6 +27,7 @@ List projects = [ 'modules:lang-mustache', 'modules:lang-painless', 'modules:transport-netty3', + 'modules:transport-netty4', 'modules:reindex', 'modules:rank-eval', 'modules:percolator', diff --git a/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java index 103e263ffa6..aae6522da12 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java @@ -430,7 +430,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length() - 1); // an offset to the end would be len 0 - int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(1, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); BytesRef singlePageOrNull = getSinglePageOrNull(slice); if (singlePageOrNull != null) { diff --git a/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java b/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java new file mode 100644 index 00000000000..db8b6825ec8 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.node; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.InternalTestCluster; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class NodeTests extends ESTestCase { + + public void testNodeName() throws IOException { + final Path tempDir = createTempDir(); + final String name = randomBoolean() ? randomAsciiOfLength(10) : null; + Settings.Builder settings = Settings.builder() + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong())) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(NetworkModule.HTTP_ENABLED.getKey(), false) + .put("discovery.type", "local") + .put("transport.type", "local") + .put(Node.NODE_DATA_SETTING.getKey(), true); + if (name != null) { + settings.put(Node.NODE_NAME_SETTING.getKey(), name); + } + try (Node node = new MockNode(settings.build(), Collections.emptyList())) { + final Settings nodeSettings = randomBoolean() ? node.settings() : node.getEnvironment().settings(); + if (name == null) { + assertThat(Node.NODE_NAME_SETTING.get(nodeSettings), equalTo(node.getNodeEnvironment().nodeId().substring(0, 7))); + } else { + assertThat(Node.NODE_NAME_SETTING.get(nodeSettings), equalTo(name)); + } + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 519e52074da..b6fddaa427c 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -20,6 +20,7 @@ package org.elasticsearch.script; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.search.lookup.LeafSearchLookup; @@ -183,6 +184,12 @@ public class MockScriptEngine implements ScriptEngineService { public void setNextVar(String name, Object value) { ctx.put(name, value); } + + @Override + public void setScorer(Scorer scorer) { + super.setScorer(scorer); + ctx.put("_score", new ScoreAccessor(scorer)); + } }; leafSearchScript.setLookup(leafLookup); return leafSearchScript; @@ -190,7 +197,7 @@ public class MockScriptEngine implements ScriptEngineService { @Override public boolean needsScores() { - return false; + return true; } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java index 3414e9eda10..16647b04a47 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESBackcompatTestCase.java @@ -242,7 +242,7 @@ public abstract class ESBackcompatTestCase extends ESIntegTestCase { protected Settings commonNodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(requiredSettings()); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3"); // run same transport / disco as external + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? "netty3" : "netty4"); // run same transport / disco as external builder.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen"); return builder.build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 73edf000b96..9c51bf23bd2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.TestUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ShardOperationFailedException; @@ -2094,11 +2095,11 @@ public abstract class ESIntegTestCase extends ESTestCase { return restClient; } - protected static RestClient createRestClient(RestClient.HttpClientConfigCallback httpClientConfigCallback) { + protected static RestClient createRestClient(RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback) { return createRestClient(httpClientConfigCallback, "http"); } - protected static RestClient createRestClient(RestClient.HttpClientConfigCallback httpClientConfigCallback, String protocol) { + protected static RestClient createRestClient(RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback, String protocol) { final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get(); final List nodes = nodeInfos.getNodes(); assertFalse(nodeInfos.hasFailures()); @@ -2111,7 +2112,7 @@ public abstract class ESIntegTestCase extends ESTestCase { hosts.add(new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), protocol)); } } - RestClient.Builder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()])); + RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()])); if (httpClientConfigCallback != null) { builder.setHttpClientConfigCallback(httpClientConfigCallback); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java index b5a5778c1e5..cde4e5f6ac9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.logging.ESLogger; @@ -51,7 +52,7 @@ final class ExternalNode implements Closeable { public static final Settings REQUIRED_SETTINGS = Settings.builder() .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen") - .put(NetworkModule.TRANSPORT_TYPE_KEY, "netty3").build(); // we need network mode for this + .put(NetworkModule.TRANSPORT_TYPE_KEY, Randomness.get().nextBoolean() ? "netty3" : "netty4").build(); // we need network mode for this private final Path path; private final Random random; diff --git a/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java b/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java index 789eb693f7f..dcb91ed441e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java +++ b/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java @@ -53,17 +53,21 @@ public class IndexSettingsModule extends AbstractModule { } public static IndexSettings newIndexSettings(Index index, Settings settings, Setting... setting) { + return newIndexSettings(index, settings, Settings.EMPTY, setting); + } + + public static IndexSettings newIndexSettings(Index index, Settings indexSetting, Settings nodeSettings, Setting... setting) { Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(settings) + .put(indexSetting) .build(); IndexMetaData metaData = IndexMetaData.builder(index.getName()).settings(build).build(); Set> settingSet = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS); if (setting.length > 0) { settingSet.addAll(Arrays.asList(setting)); } - return new IndexSettings(metaData, Settings.EMPTY, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), new IndexScopedSettings(Settings.EMPTY, settingSet)); + return new IndexSettings(metaData, nodeSettings, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), new IndexScopedSettings(Settings.EMPTY, settingSet)); } public static IndexSettings newIndexSettings(Index index, Settings settings, IndexScopedSettings indexScopedSettings) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java index 7103cda945b..d5c14935ecd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java +++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.junit.internal.AssumptionViolatedException; import org.junit.runner.Description; import org.junit.runner.notification.Failure; @@ -37,9 +37,9 @@ import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_ITERATIONS; import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_PREFIX; import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_TESTMETHOD; import static org.elasticsearch.test.ESIntegTestCase.TESTS_CLUSTER; -import static org.elasticsearch.test.rest.ESRestTestCase.REST_TESTS_BLACKLIST; -import static org.elasticsearch.test.rest.ESRestTestCase.REST_TESTS_SPEC; -import static org.elasticsearch.test.rest.ESRestTestCase.REST_TESTS_SUITE; +import static org.elasticsearch.test.rest.ESClientYamlSuiteTestCase.REST_TESTS_BLACKLIST; +import static org.elasticsearch.test.rest.ESClientYamlSuiteTestCase.REST_TESTS_SPEC; +import static org.elasticsearch.test.rest.ESClientYamlSuiteTestCase.REST_TESTS_SUITE; /** * A {@link RunListener} that emits a command you can use to re-run a failing test with the failing random seed to @@ -81,9 +81,9 @@ public class ReproduceInfoPrinter extends RunListener { GradleMessageBuilder gradleMessageBuilder = new GradleMessageBuilder(b); gradleMessageBuilder.appendAllOpts(failure.getDescription()); - //Rest tests are a special case as they allow for additional parameters - if (ESRestTestCase.class.isAssignableFrom(failure.getDescription().getTestClass())) { - gradleMessageBuilder.appendRestTestsProperties(); + // Client yaml suite tests are a special case as they allow for additional parameters + if (ESClientYamlSuiteTestCase.class.isAssignableFrom(failure.getDescription().getTestClass())) { + gradleMessageBuilder.appendClientYamlSuiteProperties(); } System.err.println(b.toString()); @@ -152,7 +152,7 @@ public class ReproduceInfoPrinter extends RunListener { return this; } - public ReproduceErrorMessageBuilder appendRestTestsProperties() { + public ReproduceErrorMessageBuilder appendClientYamlSuiteProperties() { return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC, REST_TESTS_BLACKLIST); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESClientYamlSuiteTestCase.java new file mode 100644 index 00000000000..2611e20d447 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESClientYamlSuiteTestCase.java @@ -0,0 +1,319 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest; + +import com.carrotsearch.randomizedtesting.RandomizedTest; + +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.rest.parser.RestTestParseException; +import org.elasticsearch.test.rest.parser.RestTestSuiteParser; +import org.elasticsearch.test.rest.section.DoSection; +import org.elasticsearch.test.rest.section.ExecutableSection; +import org.elasticsearch.test.rest.section.RestTestSuite; +import org.elasticsearch.test.rest.section.SkipSection; +import org.elasticsearch.test.rest.section.TestSection; +import org.elasticsearch.test.rest.spec.RestApi; +import org.elasticsearch.test.rest.spec.RestSpec; +import org.elasticsearch.test.rest.support.FileUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Runs a suite of yaml tests shared with all the official Elasticsearch clients against against an elasticsearch cluster. + */ +public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase { + + /** + * Property that allows to control which REST tests get run. Supports comma separated list of tests + * or directories that contain tests e.g. -Dtests.rest.suite=index,get,create/10_with_id + */ + public static final String REST_TESTS_SUITE = "tests.rest.suite"; + /** + * Property that allows to blacklist some of the REST tests based on a comma separated list of globs + * e.g. -Dtests.rest.blacklist=get/10_basic/* + */ + public static final String REST_TESTS_BLACKLIST = "tests.rest.blacklist"; + /** + * Property that allows to control whether spec validation is enabled or not (default true). + */ + public static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec"; + /** + * Property that allows to control where the REST spec files need to be loaded from + */ + public static final String REST_TESTS_SPEC = "tests.rest.spec"; + + public static final String REST_LOAD_PACKAGED_TESTS = "tests.rest.load_packaged"; + + private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test"; + private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api"; + + /** + * This separator pattern matches ',' except it is preceded by a '\'. + * This allows us to support ',' within paths when it is escaped with a slash. + * + * For example, the path string "/a/b/c\,d/e/f,/foo/bar,/baz" is separated to "/a/b/c\,d/e/f", "/foo/bar" and "/baz". + * + * For reference, this regular expression feature is known as zero-width negative look-behind. + * + */ + private static final String PATHS_SEPARATOR = "(? blacklistPathMatchers = new ArrayList<>(); + private static RestTestExecutionContext restTestExecutionContext; + private static RestTestExecutionContext adminExecutionContext; + + private final RestTestCandidate testCandidate; + + public ESClientYamlSuiteTestCase(RestTestCandidate testCandidate) { + this.testCandidate = testCandidate; + String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); + for (String entry : blacklist) { + this.blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry)); + } + + } + + @Override + protected void afterIfFailed(List errors) { + logger.info("Stash dump on failure [{}]", XContentHelper.toString(restTestExecutionContext.stash())); + super.afterIfFailed(errors); + } + + public static Iterable createParameters(int id, int count) throws IOException, RestTestParseException { + //parse tests only if rest test group is enabled, otherwise rest tests might not even be available on file system + List restTestCandidates = collectTestCandidates(id, count); + List objects = new ArrayList<>(); + for (RestTestCandidate restTestCandidate : restTestCandidates) { + objects.add(new Object[]{restTestCandidate}); + } + return objects; + } + + private static List collectTestCandidates(int id, int count) throws RestTestParseException, IOException { + List testCandidates = new ArrayList<>(); + FileSystem fileSystem = getFileSystem(); + // don't make a try-with, getFileSystem returns null + // ... and you can't close() the default filesystem + try { + String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH); + Map> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths); + RestTestSuiteParser restTestSuiteParser = new RestTestSuiteParser(); + //yaml suites are grouped by directory (effectively by api) + for (String api : yamlSuites.keySet()) { + List yamlFiles = new ArrayList<>(yamlSuites.get(api)); + for (Path yamlFile : yamlFiles) { + String key = api + yamlFile.getFileName().toString(); + if (mustExecute(key, id, count)) { + RestTestSuite restTestSuite = restTestSuiteParser.parse(api, yamlFile); + for (TestSection testSection : restTestSuite.getTestSections()) { + testCandidates.add(new RestTestCandidate(restTestSuite, testSection)); + } + } + } + } + } finally { + IOUtils.close(fileSystem); + } + + //sort the candidates so they will always be in the same order before being shuffled, for repeatability + Collections.sort(testCandidates, new Comparator() { + @Override + public int compare(RestTestCandidate o1, RestTestCandidate o2) { + return o1.getTestPath().compareTo(o2.getTestPath()); + } + }); + + return testCandidates; + } + + private static boolean mustExecute(String test, int id, int count) { + int hash = (int) (Math.abs((long)test.hashCode()) % count); + return hash == id; + } + + private static String[] resolvePathsProperty(String propertyName, String defaultValue) { + String property = System.getProperty(propertyName); + if (!Strings.hasLength(property)) { + return defaultValue == null ? Strings.EMPTY_ARRAY : new String[]{defaultValue}; + } else { + return property.split(PATHS_SEPARATOR); + } + } + + /** + * Returns a new FileSystem to read REST resources, or null if they + * are available from classpath. + */ + @SuppressForbidden(reason = "proper use of URL, hack around a JDK bug") + static FileSystem getFileSystem() throws IOException { + // REST suite handling is currently complicated, with lots of filtering and so on + // For now, to work embedded in a jar, return a ZipFileSystem over the jar contents. + URL codeLocation = FileUtils.class.getProtectionDomain().getCodeSource().getLocation(); + boolean loadPackaged = RandomizedTest.systemPropertyAsBoolean(REST_LOAD_PACKAGED_TESTS, true); + if (codeLocation.getFile().endsWith(".jar") && loadPackaged) { + try { + // hack around a bug in the zipfilesystem implementation before java 9, + // its checkWritable was incorrect and it won't work without write permissions. + // if we add the permission, it will open jars r/w, which is too scary! so copy to a safe r-w location. + Path tmp = Files.createTempFile(null, ".jar"); + try (InputStream in = codeLocation.openStream()) { + Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING); + } + return FileSystems.newFileSystem(new URI("jar:" + tmp.toUri()), Collections.emptyMap()); + } catch (URISyntaxException e) { + throw new IOException("couldn't open zipfilesystem: ", e); + } + } else { + return null; + } + } + + @BeforeClass + public static void initExecutionContext() throws IOException { + String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH); + RestSpec restSpec = null; + FileSystem fileSystem = getFileSystem(); + // don't make a try-with, getFileSystem returns null + // ... and you can't close() the default filesystem + try { + restSpec = RestSpec.parseFrom(fileSystem, DEFAULT_SPEC_PATH, specPaths); + } finally { + IOUtils.close(fileSystem); + } + validateSpec(restSpec); + restTestExecutionContext = new RestTestExecutionContext(restSpec); + adminExecutionContext = new RestTestExecutionContext(restSpec); + } + + protected RestTestExecutionContext getAdminExecutionContext() { + return adminExecutionContext; + } + + private static void validateSpec(RestSpec restSpec) { + boolean validateSpec = RandomizedTest.systemPropertyAsBoolean(REST_TESTS_VALIDATE_SPEC, true); + if (validateSpec) { + StringBuilder errorMessage = new StringBuilder(); + for (RestApi restApi : restSpec.getApis()) { + if (restApi.getMethods().contains("GET") && restApi.isBodySupported()) { + if (!restApi.getMethods().contains("POST")) { + errorMessage.append("\n- ").append(restApi.getName()).append(" supports GET with a body but doesn't support POST"); + } + } + } + if (errorMessage.length() > 0) { + throw new IllegalArgumentException(errorMessage.toString()); + } + } + } + + @AfterClass + public static void clearStatic() { + restTestExecutionContext = null; + adminExecutionContext = null; + } + + @Before + public void reset() throws IOException { + // admin context must be available for @After always, regardless of whether the test was blacklisted + adminExecutionContext.initClient(adminClient(), getClusterHosts()); + adminExecutionContext.clear(); + + //skip test if it matches one of the blacklist globs + for (BlacklistedPathPatternMatcher blacklistedPathMatcher : blacklistPathMatchers) { + String testPath = testCandidate.getSuitePath() + "/" + testCandidate.getTestSection().getName(); + assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher + .isSuffixMatch(testPath)); + } + //The client needs non static info to get initialized, therefore it can't be initialized in the before class + restTestExecutionContext.initClient(client(), getClusterHosts()); + restTestExecutionContext.clear(); + + //skip test if the whole suite (yaml file) is disabled + assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()), + testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion())); + //skip test if the whole suite (yaml file) is disabled + assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getTeardownSection().getSkipSection()), + testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext.esVersion())); + //skip test if test section is disabled + assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()), + testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion())); + } + + private static String buildSkipMessage(String description, SkipSection skipSection) { + StringBuilder messageBuilder = new StringBuilder(); + if (skipSection.isVersionCheck()) { + messageBuilder.append("[").append(description).append("] skipped, reason: [").append(skipSection.getReason()).append("] "); + } else { + messageBuilder.append("[").append(description).append("] skipped, reason: features ") + .append(skipSection.getFeatures()).append(" not supported"); + } + return messageBuilder.toString(); + } + + public void test() throws IOException { + //let's check that there is something to run, otherwise there might be a problem with the test section + if (testCandidate.getTestSection().getExecutableSections().size() == 0) { + throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]"); + } + + if (!testCandidate.getSetupSection().isEmpty()) { + logger.debug("start setup test [{}]", testCandidate.getTestPath()); + for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) { + doSection.execute(restTestExecutionContext); + } + logger.debug("end setup test [{}]", testCandidate.getTestPath()); + } + + restTestExecutionContext.clear(); + + try { + for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) { + executableSection.execute(restTestExecutionContext); + } + } finally { + logger.debug("start teardown test [{}]", testCandidate.getTestPath()); + for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) { + doSection.execute(restTestExecutionContext); + } + logger.debug("end teardown test [{}]", testCandidate.getTestPath()); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index e8895aa90db..573c301105a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -19,263 +19,128 @@ package org.elasticsearch.test.rest; -import com.carrotsearch.randomizedtesting.RandomizedTest; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContexts; import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; +import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.client.RestTestResponse; -import org.elasticsearch.test.rest.parser.RestTestParseException; -import org.elasticsearch.test.rest.parser.RestTestSuiteParser; -import org.elasticsearch.test.rest.section.DoSection; -import org.elasticsearch.test.rest.section.ExecutableSection; -import org.elasticsearch.test.rest.section.RestTestSuite; -import org.elasticsearch.test.rest.section.SkipSection; -import org.elasticsearch.test.rest.section.TestSection; -import org.elasticsearch.test.rest.spec.RestApi; -import org.elasticsearch.test.rest.spec.RestSpec; -import org.elasticsearch.test.rest.support.FileUtils; import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; import java.io.IOException; import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; +import javax.net.ssl.SSLContext; + import static java.util.Collections.sort; +import static java.util.Collections.unmodifiableList; /** - * Runs the clients test suite against an elasticsearch cluster. + * Superclass for tests that interact with an external test cluster using Elasticsearch's {@link RestClient}. */ -public abstract class ESRestTestCase extends ESTestCase { +public class ESRestTestCase extends ESTestCase { + public static final String TRUSTSTORE_PATH = "truststore.path"; + public static final String TRUSTSTORE_PASSWORD = "truststore.password"; /** - * Property that allows to control which REST tests get run. Supports comma separated list of tests - * or directories that contain tests e.g. -Dtests.rest.suite=index,get,create/10_with_id + * Convert the entity from a {@link Response} into a map of maps. */ - public static final String REST_TESTS_SUITE = "tests.rest.suite"; - /** - * Property that allows to blacklist some of the REST tests based on a comma separated list of globs - * e.g. -Dtests.rest.blacklist=get/10_basic/* - */ - public static final String REST_TESTS_BLACKLIST = "tests.rest.blacklist"; - /** - * Property that allows to control whether spec validation is enabled or not (default true). - */ - public static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec"; - /** - * Property that allows to control where the REST spec files need to be loaded from - */ - public static final String REST_TESTS_SPEC = "tests.rest.spec"; - - public static final String REST_LOAD_PACKAGED_TESTS = "tests.rest.load_packaged"; - - private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test"; - private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api"; - - /** - * This separator pattern matches ',' except it is preceded by a '\'. - * This allows us to support ',' within paths when it is escaped with a slash. - * - * For example, the path string "/a/b/c\,d/e/f,/foo/bar,/baz" is separated to "/a/b/c\,d/e/f", "/foo/bar" and "/baz". - * - * For reference, this regular expression feature is known as zero-width negative look-behind. - * - */ - private static final String PATHS_SEPARATOR = "(? blacklistPathMatchers = new ArrayList<>(); - private final URL[] clusterUrls; - private static RestTestExecutionContext restTestExecutionContext; - private static RestTestExecutionContext adminExecutionContext; - - private final RestTestCandidate testCandidate; - - public ESRestTestCase(RestTestCandidate testCandidate) { - this.testCandidate = testCandidate; - String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); - for (String entry : blacklist) { - this.blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry)); + public static Map entityAsMap(Response response) throws IOException { + XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); + try (XContentParser parser = xContentType.xContent().createParser(response.getEntity().getContent())) { + return parser.map(); } + } + + private final List clusterHosts; + /** + * A client for the running Elasticsearch cluster. Lazily initialized on first use. + */ + private final RestClient client; + /** + * A client for the running Elasticsearch cluster configured to take test administrative actions like remove all indexes after the test + * completes. Lazily initialized on first use. + */ + private final RestClient adminClient; + + public ESRestTestCase() { String cluster = System.getProperty("tests.rest.cluster"); if (cluster == null) { - throw new RuntimeException("Must specify tests.rest.cluster for rest tests"); + throw new RuntimeException("Must specify [tests.rest.cluster] system property with a comma delimited list of [host:port] " + + "to which to send REST requests"); } String[] stringUrls = cluster.split(","); - clusterUrls = new URL[stringUrls.length]; - int i = 0; - try { - for (String stringUrl : stringUrls) { - clusterUrls[i++] = new URL("http://" + stringUrl); + List clusterHosts = new ArrayList<>(stringUrls.length); + for (String stringUrl : stringUrls) { + int portSeparator = stringUrl.lastIndexOf(':'); + if (portSeparator < 0) { + throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); } + String host = stringUrl.substring(0, portSeparator); + int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); + clusterHosts.add(new HttpHost(host, port, getProtocol())); + } + this.clusterHosts = unmodifiableList(clusterHosts); + try { + client = buildClient(restClientSettings()); + adminClient = buildClient(restAdminSettings()); } catch (IOException e) { - throw new RuntimeException("Failed to parse cluster addresses for rest test", e); - } - } - - @Override - protected void afterIfFailed(List errors) { - logger.info("Stash dump on failure [{}]", XContentHelper.toString(restTestExecutionContext.stash())); - super.afterIfFailed(errors); - } - - public static Iterable createParameters(int id, int count) throws IOException, RestTestParseException { - //parse tests only if rest test group is enabled, otherwise rest tests might not even be available on file system - List restTestCandidates = collectTestCandidates(id, count); - List objects = new ArrayList<>(); - for (RestTestCandidate restTestCandidate : restTestCandidates) { - objects.add(new Object[]{restTestCandidate}); - } - return objects; - } - - private static List collectTestCandidates(int id, int count) throws RestTestParseException, IOException { - List testCandidates = new ArrayList<>(); - FileSystem fileSystem = getFileSystem(); - // don't make a try-with, getFileSystem returns null - // ... and you can't close() the default filesystem - try { - String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH); - Map> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths); - RestTestSuiteParser restTestSuiteParser = new RestTestSuiteParser(); - //yaml suites are grouped by directory (effectively by api) - for (String api : yamlSuites.keySet()) { - List yamlFiles = new ArrayList<>(yamlSuites.get(api)); - for (Path yamlFile : yamlFiles) { - String key = api + yamlFile.getFileName().toString(); - if (mustExecute(key, id, count)) { - RestTestSuite restTestSuite = restTestSuiteParser.parse(api, yamlFile); - for (TestSection testSection : restTestSuite.getTestSections()) { - testCandidates.add(new RestTestCandidate(restTestSuite, testSection)); - } - } - } - } - } finally { - IOUtils.close(fileSystem); - } - - //sort the candidates so they will always be in the same order before being shuffled, for repeatability - Collections.sort(testCandidates, new Comparator() { - @Override - public int compare(RestTestCandidate o1, RestTestCandidate o2) { - return o1.getTestPath().compareTo(o2.getTestPath()); - } - }); - - return testCandidates; - } - - private static boolean mustExecute(String test, int id, int count) { - int hash = (int) (Math.abs((long)test.hashCode()) % count); - return hash == id; - } - - private static String[] resolvePathsProperty(String propertyName, String defaultValue) { - String property = System.getProperty(propertyName); - if (!Strings.hasLength(property)) { - return defaultValue == null ? Strings.EMPTY_ARRAY : new String[]{defaultValue}; - } else { - return property.split(PATHS_SEPARATOR); + // Wrap the IOException so children don't have to declare a constructor just to rethrow it. + throw new RuntimeException("Error building clients", e); } } /** - * Returns a new FileSystem to read REST resources, or null if they - * are available from classpath. + * Clean up after the test case. */ - @SuppressForbidden(reason = "proper use of URL, hack around a JDK bug") - static FileSystem getFileSystem() throws IOException { - // REST suite handling is currently complicated, with lots of filtering and so on - // For now, to work embedded in a jar, return a ZipFileSystem over the jar contents. - URL codeLocation = FileUtils.class.getProtectionDomain().getCodeSource().getLocation(); - boolean loadPackaged = RandomizedTest.systemPropertyAsBoolean(REST_LOAD_PACKAGED_TESTS, true); - if (codeLocation.getFile().endsWith(".jar") && loadPackaged) { - try { - // hack around a bug in the zipfilesystem implementation before java 9, - // its checkWritable was incorrect and it won't work without write permissions. - // if we add the permission, it will open jars r/w, which is too scary! so copy to a safe r-w location. - Path tmp = Files.createTempFile(null, ".jar"); - try (InputStream in = codeLocation.openStream()) { - Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING); - } - return FileSystems.newFileSystem(new URI("jar:" + tmp.toUri()), Collections.emptyMap()); - } catch (URISyntaxException e) { - throw new IOException("couldn't open zipfilesystem: ", e); - } - } else { - return null; - } - } - - @BeforeClass - public static void initExecutionContext() throws IOException { - String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH); - RestSpec restSpec = null; - FileSystem fileSystem = getFileSystem(); - // don't make a try-with, getFileSystem returns null - // ... and you can't close() the default filesystem - try { - restSpec = RestSpec.parseFrom(fileSystem, DEFAULT_SPEC_PATH, specPaths); - } finally { - IOUtils.close(fileSystem); - } - validateSpec(restSpec); - restTestExecutionContext = new RestTestExecutionContext(restSpec); - adminExecutionContext = new RestTestExecutionContext(restSpec); - } - - protected RestTestExecutionContext getAdminExecutionContext() { - return adminExecutionContext; - } - - private static void validateSpec(RestSpec restSpec) { - boolean validateSpec = RandomizedTest.systemPropertyAsBoolean(REST_TESTS_VALIDATE_SPEC, true); - if (validateSpec) { - StringBuilder errorMessage = new StringBuilder(); - for (RestApi restApi : restSpec.getApis()) { - if (restApi.getMethods().contains("GET") && restApi.isBodySupported()) { - if (!restApi.getMethods().contains("POST")) { - errorMessage.append("\n- ").append(restApi.getName()).append(" supports GET with a body but doesn't support POST"); - } - } - } - if (errorMessage.length() > 0) { - throw new IllegalArgumentException(errorMessage.toString()); - } - } - } - @After - public void wipeCluster() throws Exception { + public final void after() throws Exception { + wipeCluster(); + logIfThereAreRunningTasks(); + closeClients(); + } + + /** + * Get a client, building it if it hasn't been built for this test. + */ + protected final RestClient client() { + return client; + } + + /** + * Get the client used for test administrative actions. Do not use this while writing a test. Only use it for cleaning up after tests. + */ + protected final RestClient adminClient() { + return adminClient; + } + + private void wipeCluster() throws IOException { // wipe indices - Map deleteIndicesArgs = new HashMap<>(); - deleteIndicesArgs.put("index", "*"); try { - adminExecutionContext.callApi("indices.delete", deleteIndicesArgs, Collections.emptyList(), Collections.emptyMap()); + adminClient().performRequest("DELETE", "*"); } catch (ResponseException e) { // 404 here just means we had no indexes if (e.getResponse().getStatusLine().getStatusCode() != 404) { @@ -284,24 +149,19 @@ public abstract class ESRestTestCase extends ESTestCase { } // wipe index templates - Map deleteTemplatesArgs = new HashMap<>(); - deleteTemplatesArgs.put("name", "*"); - adminExecutionContext.callApi("indices.delete_template", deleteTemplatesArgs, Collections.emptyList(), Collections.emptyMap()); + adminClient().performRequest("DELETE", "_template/*"); // wipe snapshots - Map deleteSnapshotsArgs = new HashMap<>(); - deleteSnapshotsArgs.put("repository", "*"); - adminExecutionContext.callApi("snapshot.delete_repository", deleteSnapshotsArgs, Collections.emptyList(), Collections.emptyMap()); + // Technically this deletes all repositories and leave the snapshots in the repository. OK. + adminClient().performRequest("DELETE", "_snapshot/*"); } /** * Logs a message if there are still running tasks. The reasoning is that any tasks still running are state the is trying to bleed into * other tests. */ - @After - public void logIfThereAreRunningTasks() throws InterruptedException, IOException { - RestTestResponse tasks = adminExecutionContext.callApi("tasks.list", emptyMap(), emptyList(), emptyMap()); - Set runningTasks = runningTasks(tasks); + private void logIfThereAreRunningTasks() throws InterruptedException, IOException { + Set runningTasks = runningTasks(adminClient().performRequest("GET", "_tasks")); // Ignore the task list API - it doens't count against us runningTasks.remove(ListTasksAction.NAME); runningTasks.remove(ListTasksAction.NAME + "[n]"); @@ -318,14 +178,8 @@ public abstract class ESRestTestCase extends ESTestCase { */ } - @AfterClass - public static void close() { - if (restTestExecutionContext != null) { - restTestExecutionContext.close(); - adminExecutionContext.close(); - restTestExecutionContext = null; - adminExecutionContext = null; - } + private void closeClients() throws IOException { + IOUtils.close(client, adminClient); } /** @@ -335,82 +189,69 @@ public abstract class ESRestTestCase extends ESTestCase { return Settings.EMPTY; } - /** Returns the REST client settings used for admin actions like cleaning up after the test has completed. */ + /** + * Returns the REST client settings used for admin actions like cleaning up after the test has completed. + */ protected Settings restAdminSettings() { return restClientSettings(); // default to the same client settings } - @Before - public void reset() throws IOException { - // admin context must be available for @After always, regardless of whether the test was blacklisted - adminExecutionContext.initClient(clusterUrls, restAdminSettings()); - adminExecutionContext.clear(); - - //skip test if it matches one of the blacklist globs - for (BlacklistedPathPatternMatcher blacklistedPathMatcher : blacklistPathMatchers) { - String testPath = testCandidate.getSuitePath() + "/" + testCandidate.getTestSection().getName(); - assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher - .isSuffixMatch(testPath)); - } - //The client needs non static info to get initialized, therefore it can't be initialized in the before class - restTestExecutionContext.initClient(clusterUrls, restClientSettings()); - restTestExecutionContext.clear(); - - //skip test if the whole suite (yaml file) is disabled - assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()), - testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion())); - //skip test if the whole suite (yaml file) is disabled - assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getTeardownSection().getSkipSection()), - testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext.esVersion())); - //skip test if test section is disabled - assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()), - testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion())); + /** + * Get the list of hosts in the cluster. + */ + protected final List getClusterHosts() { + return clusterHosts; } - private static String buildSkipMessage(String description, SkipSection skipSection) { - StringBuilder messageBuilder = new StringBuilder(); - if (skipSection.isVersionCheck()) { - messageBuilder.append("[").append(description).append("] skipped, reason: [").append(skipSection.getReason()).append("] "); - } else { - messageBuilder.append("[").append(description).append("] skipped, reason: features ") - .append(skipSection.getFeatures()).append(" not supported"); - } - return messageBuilder.toString(); + /** + * Override this to switch to testing https. + */ + protected String getProtocol() { + return "http"; } - public void test() throws IOException { - //let's check that there is something to run, otherwise there might be a problem with the test section - if (testCandidate.getTestSection().getExecutableSections().size() == 0) { - throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]"); + private RestClient buildClient(Settings settings) throws IOException { + RestClientBuilder builder = RestClient.builder(clusterHosts.toArray(new HttpHost[0])).setMaxRetryTimeoutMillis(30000) + .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setSocketTimeout(30000)); + String keystorePath = settings.get(TRUSTSTORE_PATH); + if (keystorePath != null) { + final String keystorePass = settings.get(TRUSTSTORE_PASSWORD); + if (keystorePass == null) { + throw new IllegalStateException(TRUSTSTORE_PATH + " is provided but not " + TRUSTSTORE_PASSWORD); + } + Path path = PathUtils.get(keystorePath); + if (!Files.exists(path)) { + throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file"); + } + try { + KeyStore keyStore = KeyStore.getInstance("jks"); + try (InputStream is = Files.newInputStream(path)) { + keyStore.load(is, keystorePass.toCharArray()); + } + SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build(); + SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext); + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy)); + } catch (KeyStoreException|NoSuchAlgorithmException|KeyManagementException|CertificateException e) { + throw new RuntimeException("Error setting up ssl", e); + } } - if (!testCandidate.getSetupSection().isEmpty()) { - logger.debug("start setup test [{}]", testCandidate.getTestPath()); - for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) { - doSection.execute(restTestExecutionContext); + try (ThreadContext threadContext = new ThreadContext(settings)) { + Header[] defaultHeaders = new Header[threadContext.getHeaders().size()]; + int i = 0; + for (Map.Entry entry : threadContext.getHeaders().entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); } - logger.debug("end setup test [{}]", testCandidate.getTestPath()); - } - - restTestExecutionContext.clear(); - - try { - for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) { - executableSection.execute(restTestExecutionContext); - } - } finally { - logger.debug("start teardown test [{}]", testCandidate.getTestPath()); - for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) { - doSection.execute(restTestExecutionContext); - } - logger.debug("end teardown test [{}]", testCandidate.getTestPath()); + builder.setDefaultHeaders(defaultHeaders); } + return builder.build(); } @SuppressWarnings("unchecked") - public Set runningTasks(RestTestResponse response) throws IOException { + private Set runningTasks(Response response) throws IOException { Set runningTasks = new HashSet<>(); - Map nodes = (Map) response.evaluate("nodes"); + + Map nodes = (Map) entityAsMap(response).get("nodes"); for (Map.Entry node : nodes.entrySet()) { Map nodeInfo = (Map) node.getValue(); Map nodeTasks = (Map) nodeInfo.get("tasks"); @@ -421,4 +262,5 @@ public abstract class ESRestTestCase extends ESTestCase { } return runningTasks; } + } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java index 3fc924176f0..f291e0bbeec 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java @@ -28,25 +28,19 @@ import java.util.Map; public class FakeRestRequest extends RestRequest { private final Map headers; - - private final Map params; - private final BytesReference content; - private final Method method; - private final String path; public FakeRestRequest() { this(new HashMap<>(), new HashMap<>(), null, Method.GET, "/"); } private FakeRestRequest(Map headers, Map params, BytesReference content, Method method, String path) { + super(params, path); this.headers = headers; - this.params = params; this.content = content; this.method = method; - this.path = path; } @Override @@ -56,12 +50,7 @@ public class FakeRestRequest extends RestRequest { @Override public String uri() { - return path; - } - - @Override - public String rawPath() { - return path; + return rawPath(); } @Override @@ -84,31 +73,8 @@ public class FakeRestRequest extends RestRequest { return headers.entrySet(); } - @Override - public boolean hasParam(String key) { - return params.containsKey(key); - } - - @Override - public String param(String key) { - return params.get(key); - } - - @Override - public String param(String key, String defaultValue) { - String value = params.get(key); - if (value == null) { - return defaultValue; - } - return value; - } - - @Override - public Map params() { - return params; - } - public static class Builder { + private Map headers = new HashMap<>(); private Map params = new HashMap<>(); @@ -147,6 +113,7 @@ public class FakeRestRequest extends RestRequest { public FakeRestRequest build() { return new FakeRestRequest(headers, params, content, method, path); } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java index d7295e1dca7..59d45bc0a88 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java @@ -18,19 +18,18 @@ */ package org.elasticsearch.test.rest; +import org.apache.http.HttpHost; import org.elasticsearch.Version; -import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.rest.client.RestTestClient; import org.elasticsearch.test.rest.client.RestTestResponse; +import org.elasticsearch.test.rest.client.RestTestResponseException; import org.elasticsearch.test.rest.spec.RestSpec; -import java.io.Closeable; import java.io.IOException; -import java.net.URL; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,7 +40,7 @@ import java.util.Map; * Caches the last obtained test response and allows to stash part of it within variables * that can be used as input values in following requests. */ -public class RestTestExecutionContext implements Closeable { +public class RestTestExecutionContext { private static final ESLogger logger = Loggers.getLogger(RestTestExecutionContext.class); @@ -62,7 +61,7 @@ public class RestTestExecutionContext implements Closeable { * Saves the obtained response in the execution context. */ public RestTestResponse callApi(String apiName, Map params, List> bodies, - Map headers) throws IOException { + Map headers) throws IOException { //makes a copy of the parameters before modifying them for this specific request HashMap requestParams = new HashMap<>(params); for (Map.Entry entry : requestParams.entrySet()) { @@ -72,15 +71,15 @@ public class RestTestExecutionContext implements Closeable { } String body = actualBody(bodies); - try { response = callApiInternal(apiName, requestParams, body, headers); + return response; + } catch(RestTestResponseException e) { + response = e.getRestTestResponse(); + throw e; + } finally { //we always stash the last response body stash.stashValue("body", response.getBody()); - return response; - } catch(ResponseException e) { - response = new RestTestResponse(e); - throw e; } } @@ -119,10 +118,8 @@ public class RestTestExecutionContext implements Closeable { /** * Creates the embedded REST client when needed. Needs to be called before each test. */ - public void initClient(URL[] urls, Settings settings) throws IOException { - if (restTestClient == null) { - restTestClient = new RestTestClient(restSpec, settings, urls); - } + public void initClient(RestClient client, List hosts) throws IOException { + restTestClient = new RestTestClient(restSpec, client, hosts); } /** @@ -145,13 +142,4 @@ public class RestTestExecutionContext implements Closeable { return restTestClient.getEsVersion(); } - /** - * Closes the execution context and releases the underlying resources - */ - @Override - public void close() { - if (restTestClient != null) { - restTestClient.close(); - } - } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java index c97ca7cd2fd..2fb0374d048 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java @@ -19,43 +19,27 @@ package org.elasticsearch.test.rest.client; import com.carrotsearch.randomizedtesting.RandomizedTest; + import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; -import org.apache.http.ssl.SSLContexts; -import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.SSLSocketFactoryHttpConfigCallback; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.test.rest.spec.RestApi; import org.elasticsearch.test.rest.spec.RestSpec; -import javax.net.ssl.SSLContext; -import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -69,35 +53,29 @@ import java.util.Set; * Wraps a {@link RestClient} instance used to send the REST requests. * Holds the {@link RestSpec} used to translate api calls into REST calls */ -public class RestTestClient implements Closeable { - - public static final String PROTOCOL = "protocol"; - public static final String TRUSTSTORE_PATH = "truststore.path"; - public static final String TRUSTSTORE_PASSWORD = "truststore.password"; - +public class RestTestClient { private static final ESLogger logger = Loggers.getLogger(RestTestClient.class); - //query_string params that don't need to be declared in the spec, thay are supported by default + //query_string params that don't need to be declared in the spec, they are supported by default private static final Set ALWAYS_ACCEPTED_QUERY_STRING_PARAMS = Sets.newHashSet("pretty", "source", "filter_path"); private final RestSpec restSpec; private final RestClient restClient; private final Version esVersion; - public RestTestClient(RestSpec restSpec, Settings settings, URL[] urls) throws IOException { - assert urls.length > 0; + public RestTestClient(RestSpec restSpec, RestClient restClient, List hosts) throws IOException { + assert hosts.size() > 0; this.restSpec = restSpec; - this.restClient = createRestClient(urls, settings); - this.esVersion = readAndCheckVersion(urls); - logger.info("REST client initialized {}, elasticsearch version: [{}]", urls, esVersion); + this.restClient = restClient; + this.esVersion = readAndCheckVersion(hosts); } - private Version readAndCheckVersion(URL[] urls) throws IOException { + private Version readAndCheckVersion(List hosts) throws IOException { RestApi restApi = restApi("info"); assert restApi.getPaths().size() == 1; assert restApi.getMethods().size() == 1; String version = null; - for (URL ignored : urls) { + for (HttpHost ignored : hosts) { //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster String method = restApi.getMethods().get(0); String endpoint = restApi.getPaths().get(0); @@ -135,11 +113,15 @@ public class RestTestClient implements Closeable { String path = "/"+ Objects.requireNonNull(queryStringParams.remove("path"), "Path must be set to use raw request"); HttpEntity entity = null; if (body != null && body.length() > 0) { - entity = new StringEntity(body, RestClient.JSON_CONTENT_TYPE); + entity = new StringEntity(body, ContentType.APPLICATION_JSON); } // And everything else is a url parameter! - Response response = restClient.performRequest(method, path, queryStringParams, entity); - return new RestTestResponse(response); + try { + Response response = restClient.performRequest(method, path, queryStringParams, entity); + return new RestTestResponse(response); + } catch(ResponseException e) { + throw new RestTestResponseException(e); + } } List ignores = new ArrayList<>(); @@ -200,7 +182,7 @@ public class RestTestClient implements Closeable { requestMethod = "GET"; } else { requestMethod = RandomizedTest.randomFrom(supportedMethods); - requestBody = new StringEntity(body, RestClient.JSON_CONTENT_TYPE); + requestBody = new StringEntity(body, ContentType.APPLICATION_JSON); } } else { if (restApi.isBodyRequired()) { @@ -242,14 +224,13 @@ public class RestTestClient implements Closeable { logger.debug("calling api [{}]", apiName); try { - Response response = restClient.performRequest(requestMethod, requestPath, - queryStringParams, requestBody, requestHeaders); + Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders); return new RestTestResponse(response); } catch(ResponseException e) { if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) { - return new RestTestResponse(e); + return new RestTestResponse(e.getResponse()); } - throw e; + throw new RestTestResponseException(e); } } @@ -260,56 +241,4 @@ public class RestTestClient implements Closeable { } return restApi; } - - private static RestClient createRestClient(URL[] urls, Settings settings) throws IOException { - String protocol = settings.get(PROTOCOL, "http"); - HttpHost[] hosts = new HttpHost[urls.length]; - for (int i = 0; i < hosts.length; i++) { - URL url = urls[i]; - hosts[i] = new HttpHost(url.getHost(), url.getPort(), protocol); - } - RestClient.Builder builder = RestClient.builder(hosts).setMaxRetryTimeoutMillis(30000) - .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setSocketTimeout(30000)); - - String keystorePath = settings.get(TRUSTSTORE_PATH); - if (keystorePath != null) { - final String keystorePass = settings.get(TRUSTSTORE_PASSWORD); - if (keystorePass == null) { - throw new IllegalStateException(TRUSTSTORE_PATH + " is provided but not " + TRUSTSTORE_PASSWORD); - } - Path path = PathUtils.get(keystorePath); - if (!Files.exists(path)) { - throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file"); - } - try { - KeyStore keyStore = KeyStore.getInstance("jks"); - try (InputStream is = Files.newInputStream(path)) { - keyStore.load(is, keystorePass.toCharArray()); - } - SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build(); - SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslcontext); - builder.setHttpClientConfigCallback(new SSLSocketFactoryHttpConfigCallback(sslConnectionSocketFactory)); - } catch (KeyStoreException|NoSuchAlgorithmException|KeyManagementException|CertificateException e) { - throw new RuntimeException(e); - } - } - - try (ThreadContext threadContext = new ThreadContext(settings)) { - Header[] defaultHeaders = new Header[threadContext.getHeaders().size()]; - int i = 0; - for (Map.Entry entry : threadContext.getHeaders().entrySet()) { - defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); - } - builder.setDefaultHeaders(defaultHeaders); - } - return builder.build(); - } - - /** - * Closes the REST client and the underlying http client - */ - @Override - public void close() { - IOUtils.closeWhileHandlingException(restClient); - } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java index 4644b87b8e7..9149824c33c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java @@ -20,9 +20,7 @@ package org.elasticsearch.test.rest.client; import org.apache.http.client.methods.HttpHead; import org.apache.http.util.EntityUtils; -import org.apache.lucene.util.IOUtils; import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.rest.Stash; @@ -40,7 +38,7 @@ public class RestTestResponse { private final String body; private ObjectPath parsedResponse; - public RestTestResponse(Response response) throws IOException { + RestTestResponse(Response response) throws IOException { this.response = response; if (response.getEntity() != null) { try { @@ -48,8 +46,6 @@ public class RestTestResponse { } catch (IOException e) { EntityUtils.consumeQuietly(response.getEntity()); throw new RuntimeException(e); - } finally { - IOUtils.closeWhileHandlingException(response); } } else { this.body = null; @@ -57,12 +53,6 @@ public class RestTestResponse { parseResponseBody(); } - public RestTestResponse(ResponseException responseException) throws IOException { - this.response = responseException.getResponse(); - this.body = responseException.getResponseBody(); - parseResponseBody(); - } - private void parseResponseBody() throws IOException { if (body != null) { String contentType = response.getHeader("Content-Type"); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponseException.java new file mode 100644 index 00000000000..2fc93a91088 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponseException.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.client; + +import org.elasticsearch.client.ResponseException; + +import java.io.IOException; + +/** + * Exception obtained from a REST call in case the response code indicated an error. Eagerly reads the response body into a string + * for later optional parsing. Supports parsing the response body when needed and returning specific values extracted from it. + */ +public class RestTestResponseException extends IOException { + + private final RestTestResponse restTestResponse; + private final ResponseException responseException; + + RestTestResponseException(ResponseException responseException) throws IOException { + super(responseException); + this.responseException = responseException; + this.restTestResponse = new RestTestResponse(responseException.getResponse()); + } + + /** + * Exposes the obtained response body + */ + public RestTestResponse getRestTestResponse() { + return restTestResponse; + } + + /** + * Exposes the origina {@link ResponseException}. Note that the entity will always be null as it + * gets eagerly consumed and exposed through {@link #getRestTestResponse()}. + */ + public ResponseException getResponseException() { + return responseException; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java index 2547d6becea..20b4f66f06c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java @@ -18,13 +18,13 @@ */ package org.elasticsearch.test.rest.section; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.rest.RestTestExecutionContext; import org.elasticsearch.test.rest.client.RestTestResponse; +import org.elasticsearch.test.rest.client.RestTestResponseException; import java.io.IOException; import java.util.HashMap; @@ -102,8 +102,8 @@ public class DoSection implements ExecutableSection { } fail(formatStatusCodeMessage(restTestResponse, catchStatusCode)); } - } catch(ResponseException e) { - RestTestResponse restTestResponse = new RestTestResponse(e); + } catch(RestTestResponseException e) { + RestTestResponse restTestResponse = e.getRestTestResponse(); if (!Strings.hasLength(catchParam)) { fail(formatStatusCodeMessage(restTestResponse, "2xx")); } else if (catches.containsKey(catchParam)) { @@ -111,7 +111,7 @@ public class DoSection implements ExecutableSection { } else if (catchParam.length() > 2 && catchParam.startsWith("/") && catchParam.endsWith("/")) { //the text of the error message matches regular expression assertThat(formatStatusCodeMessage(restTestResponse, "4xx|5xx"), - e.getResponse().getStatusLine().getStatusCode(), greaterThanOrEqualTo(400)); + e.getResponseException().getResponse().getStatusLine().getStatusCode(), greaterThanOrEqualTo(400)); Object error = executionContext.response("error"); assertThat("error was expected in the response", error, notNullValue()); //remove delimiters from regex diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java index 3c78d432f19..98bce54082f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java @@ -24,6 +24,8 @@ import org.elasticsearch.test.ESIntegTestCase; import java.util.Arrays; import java.util.List; +import static java.util.Collections.unmodifiableList; + /** * Allows to register additional features supported by the tests runner. * This way any runner can add extra features and use proper skip sections to avoid @@ -34,8 +36,12 @@ import java.util.List; */ public final class Features { - private static final List SUPPORTED = - Arrays.asList("stash_in_path", "groovy_scripting", "headers", "embedded_stash_key", "yaml"); + private static final List SUPPORTED = unmodifiableList(Arrays.asList( + "embedded_stash_key", + "groovy_scripting", + "headers", + "stash_in_path", + "yaml")); private Features() { diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 42275c75e5a..58cc45a75a1 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -21,6 +21,8 @@ package org.elasticsearch.transport; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -28,6 +30,8 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; @@ -41,16 +45,19 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -75,24 +82,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { public void setUp() throws Exception { super.setUp(); threadPool = new TestThreadPool(getClass().getName()); - serviceA = build( - Settings.builder() - .put("name", "TS_A") - .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "") - .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING") - .build(), - version0); - serviceA.acceptIncomingRequests(); + serviceA = buildService("TS_A", version0); nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0); // serviceA.setLocalNode(nodeA); - serviceB = build( - Settings.builder() - .put("name", "TS_B") - .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "") - .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING") - .build(), - version1); - serviceB.acceptIncomingRequests(); + serviceB = buildService("TS_B", version1); nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1); //serviceB.setLocalNode(nodeB); // wait till all nodes are properly connected and the event has been sent, so tests in this class @@ -131,6 +124,18 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { serviceB.removeConnectionListener(waitForConnection); } + private MockTransportService buildService(final String name, final Version version) { + MockTransportService service = build( + Settings.builder() + .put("name", name) + .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "") + .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING") + .build(), + version); + service.acceptIncomingRequests(); + return service; + } + @Override @After public void tearDown() throws Exception { @@ -483,6 +488,124 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase { assertThat(latch.await(5, TimeUnit.SECONDS), equalTo(true)); } + @TestLogging("transport:DEBUG") + public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierException, InterruptedException { + Set sendingErrors = ConcurrentCollections.newConcurrentSet(); + Set responseErrors = ConcurrentCollections.newConcurrentSet(); + serviceA.registerRequestHandler("test", TestRequest::new, + randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC, (request, channel) -> { + try { + channel.sendResponse(new TestResponse()); + } catch (Exception e) { + logger.info("caught exception while responding", e); + responseErrors.add(e); + } + }); + final TransportRequestHandler ignoringRequestHandler = (request, channel) -> { + try { + channel.sendResponse(new TestResponse()); + } catch (Exception e) { + // we don't really care what's going on B, we're testing through A + logger.trace("caught exception while res ponding from node B", e); + } + }; + serviceB.registerRequestHandler("test", TestRequest::new, ThreadPool.Names.SAME, ignoringRequestHandler); + + int halfSenders = scaledRandomIntBetween(3, 10); + final CyclicBarrier go = new CyclicBarrier(halfSenders * 2 + 1); + final CountDownLatch done = new CountDownLatch(halfSenders * 2); + for (int i = 0; i < halfSenders; i++) { + // B senders just generated activity so serciveA can respond, we don't test what's going on there + final int sender = i; + threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + logger.trace("caught exception while sending from B", e); + } + + @Override + protected void doRun() throws Exception { + go.await(); + for (int iter = 0; iter < 10; iter++) { + PlainActionFuture listener = new PlainActionFuture<>(); + final String info = sender + "_B_" + iter; + serviceB.sendRequest(nodeA, "test", new TestRequest(info), + new ActionListenerResponseHandler<>(listener, TestResponse::new)); + try { + listener.actionGet(); + + } catch (Exception e) { + logger.trace("caught exception while sending to node {}", e, nodeA); + } + } + } + + @Override + public void onAfter() { + done.countDown(); + } + }); + } + + for (int i = 0; i < halfSenders; i++) { + final int sender = i; + threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + logger.error("unexpected error", e); + sendingErrors.add(e); + } + + @Override + protected void doRun() throws Exception { + go.await(); + for (int iter = 0; iter < 10; iter++) { + PlainActionFuture listener = new PlainActionFuture<>(); + final String info = sender + "_" + iter; + final DiscoveryNode node = nodeB; // capture now + serviceA.sendRequest(node, "test", new TestRequest(info), + new ActionListenerResponseHandler<>(listener, TestResponse::new)); + try { + listener.actionGet(); + } catch (ConnectTransportException e) { + // ok! + } catch (Exception e) { + logger.error("caught exception while sending to node {}", e, node); + sendingErrors.add(e); + } + } + } + + @Override + public void onAfter() { + done.countDown(); + } + }); + } + go.await(); + for (int i = 0; i <= 10; i++) { + if (i % 3 == 0) { + // simulate restart of nodeB + serviceB.close(); + MockTransportService newService = buildService("TS_B", version1); + newService.registerRequestHandler("test", TestRequest::new, ThreadPool.Names.SAME, ignoringRequestHandler); + serviceB = newService; + nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1); + serviceB.connectToNode(nodeA); + serviceA.connectToNode(nodeB); + } else if (serviceA.nodeConnected(nodeB)) { + serviceA.disconnectFromNode(nodeB); + } else { + serviceA.connectToNode(nodeB); + } + } + + done.await(); + + assertThat("found non connection errors while sending", sendingErrors, empty()); + assertThat("found non connection errors while responding", responseErrors, empty()); + } + public void testNotifyOnShutdown() throws Exception { final CountDownLatch latch2 = new CountDownLatch(1); diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java index eb4613f9e12..44a2bbd2166 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java @@ -43,6 +43,7 @@ import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -146,8 +147,8 @@ public class MockTcpTransport extends TcpTransport output.write(minimalHeader); output.writeInt(msgSize); output.write(buffer); - BytesReference bytes = output.bytes(); - if (validateMessageHeader(bytes)) { + final BytesReference bytes = output.bytes(); + if (TcpTransport.validateMessageHeader(bytes)) { InetSocketAddress remoteAddress = (InetSocketAddress) socket.getRemoteSocketAddress(); messageReceived(bytes.slice(TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE, msgSize), mockChannel, mockChannel.profile, remoteAddress, msgSize); @@ -349,5 +350,6 @@ public class MockTcpTransport extends TcpTransport protected Version getCurrentVersion() { return mockVersion; } + } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java index a198ef77956..2b0f551dbb7 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java @@ -37,4 +37,5 @@ public class MockTransportClient extends TransportClient { public MockTransportClient(Settings settings, Collection> plugins) { super(settings, DEFAULT_SETTINGS, plugins); } + }